diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 8e09808f08bba27..fb11c82bb96d9cd 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,13 +1,10 @@ { - "image": "ghcr.io/python/devcontainer:2025.05.29.15334414373", + "image": "ghcr.io/python/devcontainer:latest", "onCreateCommand": [ // Install common tooling. "dnf", "install", "-y", - "which", - "zsh", - "fish", // For umask fix below. "/usr/bin/setfacl" ], diff --git a/.devcontainer/wasi/devcontainer.json b/.devcontainer/wasi/devcontainer.json new file mode 100644 index 000000000000000..4266144ce47639e --- /dev/null +++ b/.devcontainer/wasi/devcontainer.json @@ -0,0 +1,73 @@ +{ + "image": "ghcr.io/python/wasicontainer:latest", + "onCreateCommand": [ + // Install common tooling. + "dnf", + "install", + "-y", + // For umask fix below. + "/usr/bin/setfacl" + ], + "updateContentCommand": { + // Using the shell for `nproc` usage. + "python": "python3 Tools/wasm/wasi build --quiet -- --with-pydebug -C" + }, + "postCreateCommand": { + // https://github.com/orgs/community/discussions/26026 + "umask fix: workspace": ["sudo", "setfacl", "-bnR", "."], + "umask fix: /tmp": ["sudo", "setfacl", "-bnR", "/tmp"] + }, + "customizations": { + "vscode": { + "extensions": [ + // Highlighting for Parser/Python.asdl. + "brettcannon.zephyr-asdl", + // Highlighting for configure.ac. + "maelvalais.autoconf", + // C auto-complete. + "ms-vscode.cpptools", + // Python auto-complete. + "ms-python.python" + ], + "settings": { + "C_Cpp.default.compilerPath": "/usr/bin/clang", + "C_Cpp.default.cStandard": "c11", + "C_Cpp.default.defines": [ + "CONFIG_64", + "Py_BUILD_CORE" + ], + "C_Cpp.default.includePath": [ + "${workspaceFolder}/*", + "${workspaceFolder}/Include/**" + ], + // https://github.com/microsoft/vscode-cpptools/issues/10732 + "C_Cpp.errorSquiggles": "disabled", + "editor.insertSpaces": true, + "editor.rulers": [ + 80 + ], + "editor.tabSize": 4, + "editor.trimAutoWhitespace": true, + "files.associations": { + "*.h": "c" + }, + "files.encoding": "utf8", + "files.eol": "\n", + "files.insertFinalNewline": true, + "files.trimTrailingWhitespace": true, + "python.analysis.diagnosticSeverityOverrides": { + // Complains about shadowing the stdlib w/ the stdlib. + "reportShadowedImports": "none", + // Doesn't like _frozen_importlib. + "reportMissingImports": "none" + }, + "python.analysis.extraPaths": [ + "Lib" + ], + "[restructuredtext]": { + "editor.tabSize": 3 + } + } + } + } +} diff --git a/.editorconfig b/.editorconfig index 25bc5935258bd14..ab1f7ce8425769e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,11 +1,11 @@ root = true -[*.{py,c,cpp,h,js,rst,md,yml,yaml,gram}] +[*.{py,c,cpp,h,js,rst,md,yml,yaml,toml,gram}] trim_trailing_whitespace = true insert_final_newline = true indent_style = space -[*.{py,c,cpp,h,gram}] +[*.{py,c,cpp,h,toml,gram}] indent_size = 4 [*.rst] diff --git a/.gitattributes b/.gitattributes index 5682b9150a36e2d..4d52f900b93179e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -34,6 +34,9 @@ Lib/test/xmltestdata/* noeol Lib/venv/scripts/common/activate text eol=lf Lib/venv/scripts/posix/* text eol=lf +# Prevent GitHub's web conflict editor from converting LF to CRLF +*.rst text eol=lf + # CRLF files [attr]dos text eol=crlf @@ -68,6 +71,7 @@ PCbuild/readme.txt dos **/clinic/*.cpp.h generated **/clinic/*.h.h generated *_db.h generated +Doc/_static/tachyon-example-*.html generated Doc/c-api/lifecycle.dot.svg generated Doc/data/stable_abi.dat generated Doc/library/token-list.inc generated @@ -80,16 +84,22 @@ Include/internal/pycore_uop_ids.h generated Include/internal/pycore_uop_metadata.h generated Include/opcode.h generated Include/opcode_ids.h generated +Include/slots_generated.h generated Include/token.h generated Lib/_opcode_metadata.py generated +Lib/idlelib/help.html generated Lib/keyword.py generated +Lib/pydoc_data/topics.py generated +Lib/pydoc_data/module_docs.py generated Lib/test/certdata/*.pem generated Lib/test/certdata/*.0 generated Lib/test/levenshtein_examples.json generated Lib/test/test_stable_abi_ctypes.py generated +Lib/test/test_zoneinfo/data/*.json generated Lib/token.py generated Misc/sbom.spdx.json generated -Objects/typeslots.inc generated +Modules/_testinternalcapi/test_cases.c.h generated +Modules/_testinternalcapi/test_targets.h generated PC/python3dll.c generated Parser/parser.c generated Parser/token.c generated @@ -99,7 +109,11 @@ Python/executor_cases.c.h generated Python/generated_cases.c.h generated Python/optimizer_cases.c.h generated Python/opcode_targets.h generated +Python/record_functions.c.h generated +Python/slots_generated.c generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated aclocal.m4 generated configure generated +*.min.js generated +package-lock.json generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 02a7b5d45b46274..ecd24bb94d75641 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,348 +1,652 @@ # See https://help.github.com/articles/about-codeowners/ -# for more info about CODEOWNERS file +# for further details about the .github/CODEOWNERS file. # It uses the same pattern rule for gitignore file # https://git-scm.com/docs/gitignore#_pattern_format - -# GitHub -.github/** @ezio-melotti @hugovk @AA-Turner - -# pre-commit -.pre-commit-config.yaml @hugovk @AlexWaygood +# Notably, a later match overrides earlier matches, so order matters. +# If using a wildcard pattern, try to be as specific as possible to avoid +# matching unintended files or overriding previous entries. +# To exclude a file from ownership, add a line with only the file. +# See the exclusions section at the end of the file for examples. + +# ======= +# Purpose +# ======= +# +# An entry in this file does not imply 'ownership', despite the name of the +# file, but instead that those listed take an interest in that part of the +# project and will automatically be added as reviewers to PRs that affect +# the matching files. +# See also the Experts Index in the Python Developer's Guide: +# https://devguide.python.org/core-developers/experts/ +# +# ========= +# Structure +# ========= +# +# The CODEOWNERS file is organised by topic area. +# Please add new entries in alphabetical order within the relevant section. +# Where possible, keep related files together. For example, documentation, +# code, and tests for a given item should all be listed in the same place. +# +# GitHub usernames should be aligned to column 31, or the next multiple +# of three if the relevant paths are too long to fit. +# +# Top-level sections are: +# +# * Buildbots, Continuous Integration, and Testing +# project-wide configuration files, internal tools for use in CI, +# linting. +# * Build System +# the Makefile, autoconf, and other autotools files. +# * Documentation +# broader sections of documentation, documentation tools +# * Internal Tools & Data +# internal tools, integration with external systems, +# entries that don't fit elsewhere +# * Platform Support +# relating to support for specific platforms +# * Interpreter Core +# the grammar, parser, compiler, interpreter, etc. +# * Standard Library +# standard library modules (from both Lib and Modules) +# and related files (such as their tests and docs) +# * Exclusions +# exclusions from .github/CODEOWNERS should go at the very end +# because the final matching pattern will take precedence. + +# ---------------------------------------------------------------------------- +# Buildbots, Continuous Integration, and Testing +# ---------------------------------------------------------------------------- + +# Azure Pipelines +.azure-pipelines/ @AA-Turner + +# GitHub & related scripts +.github/ @ezio-melotti @hugovk @AA-Turner @webknjaz @itamaro +Tools/build/compute-changes.py @AA-Turner @hugovk @webknjaz +Lib/test/test_tools/test_compute_changes.py @AA-Turner @hugovk @webknjaz +Tools/build/verify_ensurepip_wheels.py @AA-Turner @pfmoore @pradyunsg + +# Pre-commit +.pre-commit-config.yaml @hugovk .ruff.toml @hugovk @AlexWaygood @AA-Turner -# Build system -configure* @erlend-aasland @corona10 -Makefile.pre.in @erlend-aasland -Modules/Setup* @erlend-aasland +# Patchcheck +Tools/patchcheck/ @AA-Turner @itamaro -# argparse -**/*argparse* @savannahostrowski -# asyncio -**/*asyncio* @1st1 @asvetlov @kumaraditya303 @willingc +# ---------------------------------------------------------------------------- +# Build System +# ---------------------------------------------------------------------------- -# Core -**/*context* @1st1 -**/*genobject* @markshannon -**/*hamt* @1st1 -**/*jit* @brandtbucher @savannahostrowski @diegorusso -Python/perf_jit_trampoline.c # Exclude the owners of "**/*jit*", above. -Objects/set* @rhettinger -Objects/dict* @methane @markshannon -Objects/typevarobject.c @JelleZijlstra -Objects/unionobject.c @JelleZijlstra -Objects/type* @markshannon -Objects/codeobject.c @markshannon -Objects/frameobject.c @markshannon -Objects/call.c @markshannon -Objects/object.c @ZeroIntensity -Python/ceval*.c @markshannon -Python/ceval*.h @markshannon -Python/codegen.c @markshannon @iritkatriel -Python/compile.c @markshannon @iritkatriel -Python/assemble.c @markshannon @iritkatriel -Python/flowgraph.c @markshannon @iritkatriel -Python/instruction_sequence.c @iritkatriel -Python/bytecodes.c @markshannon -Python/optimizer*.c @markshannon -Python/optimizer_analysis.c @Fidget-Spinner @tomasr8 -Python/optimizer_bytecodes.c @Fidget-Spinner @tomasr8 -Python/optimizer_symbols.c @tomasr8 -Python/symtable.c @JelleZijlstra @carljm -Lib/_pyrepl/* @pablogsal @lysnikolaou @ambv -Lib/test/test_patma.py @brandtbucher -Lib/test/test_type_*.py @JelleZijlstra -Lib/test/test_capi/test_misc.py @markshannon -Lib/test/test_pyrepl/* @pablogsal @lysnikolaou @ambv -Tools/c-analyzer/ @ericsnowcurrently +# Autotools +configure* @erlend-aasland @corona10 @AA-Turner @emmatyping @itamaro +Makefile.pre.in @erlend-aasland @AA-Turner @emmatyping @itamaro +Modules/makesetup @erlend-aasland @AA-Turner @emmatyping @itamaro +Modules/Setup* @erlend-aasland @AA-Turner @emmatyping @itamaro +Tools/build/regen-configure.sh @AA-Turner @itamaro -# dbm -**/*dbm* @corona10 @erlend-aasland @serhiy-storchaka - -# Doc/ tools -Doc/conf.py @AA-Turner @hugovk -Doc/Makefile @AA-Turner @hugovk -Doc/make.bat @AA-Turner @hugovk -Doc/requirements.txt @AA-Turner @hugovk -Doc/_static/** @AA-Turner @hugovk -Doc/tools/** @AA-Turner @hugovk - -# runtime state/lifecycle -**/*pylifecycle* @ericsnowcurrently @ZeroIntensity -**/*pystate* @ericsnowcurrently @ZeroIntensity -**/*preconfig* @ericsnowcurrently -**/*initconfig* @ericsnowcurrently -**/*pathconfig* @ericsnowcurrently -**/*sysmodule* @ericsnowcurrently -**/*bltinmodule* @ericsnowcurrently -**/*gil* @ericsnowcurrently -Include/internal/pycore_runtime.h @ericsnowcurrently -Include/internal/pycore_interp.h @ericsnowcurrently -Include/internal/pycore_tstate.h @ericsnowcurrently -Include/internal/pycore_*_state.h @ericsnowcurrently -Include/internal/pycore_*_init.h @ericsnowcurrently -Include/internal/pycore_atexit.h @ericsnowcurrently -Include/internal/pycore_freelist.h @ericsnowcurrently -Include/internal/pycore_global_objects.h @ericsnowcurrently -Include/internal/pycore_obmalloc.h @ericsnowcurrently -Include/internal/pycore_pymem.h @ericsnowcurrently -Include/internal/pycore_stackref.h @Fidget-Spinner -Modules/main.c @ericsnowcurrently -Programs/_bootstrap_python.c @ericsnowcurrently -Programs/python.c @ericsnowcurrently -Tools/build/generate_global_objects.py @ericsnowcurrently - -# Initialization -Doc/library/sys_path_init.rst @FFY00 -Doc/c-api/init_config.rst @FFY00 +# generate-build-details +Tools/build/generate-build-details.py @FFY00 +Lib/test/test_build_details.py @FFY00 -# getpath -**/*getpath* @FFY00 -# site -**/*site.py @FFY00 -Doc/library/site.rst @FFY00 +# ---------------------------------------------------------------------------- +# Documentation +# ---------------------------------------------------------------------------- -# Exceptions -Lib/test/test_except*.py @iritkatriel -Objects/exceptions.c @iritkatriel +# Internal Docs +InternalDocs/ @AA-Turner -# Hashing & cryptographic primitives -**/*hashlib* @gpshead @tiran @picnixz -**/*hashopenssl* @gpshead @tiran @picnixz -**/*pyhash* @gpshead @tiran @picnixz -Modules/*blake* @gpshead @tiran @picnixz -Modules/*md5* @gpshead @tiran @picnixz -Modules/*sha* @gpshead @tiran @picnixz -Modules/_hacl/** @gpshead @picnixz -**/*hmac* @gpshead @picnixz +# Tools, Configuration, etc +Doc/Makefile @AA-Turner @hugovk @StanFromIreland +Doc/_static/ @AA-Turner @hugovk @StanFromIreland +Doc/conf.py @AA-Turner @hugovk @StanFromIreland +Doc/make.bat @AA-Turner @hugovk @StanFromIreland +Doc/requirements.txt @AA-Turner @hugovk @StanFromIreland +Doc/tools/ @AA-Turner @hugovk @StanFromIreland -# libssl -**/*ssl* @gpshead @picnixz +# PR Previews +.readthedocs.yml @AA-Turner -# logging -**/*logging* @vsajip +# Sections +Doc/c-api/ @ZeroIntensity +Doc/reference/ @willingc @AA-Turner +Doc/whatsnew/ @AA-Turner -# venv -**/*venv* @vsajip @FFY00 -# Launcher -/PC/launcher.c @vsajip +# ---------------------------------------------------------------------------- +# Internal Tools and Data +# ---------------------------------------------------------------------------- -# HTML -/Lib/html/ @ezio-melotti -/Lib/_markupbase.py @ezio-melotti -/Lib/test/test_html*.py @ezio-melotti -/Tools/build/parse_html5_entities.py @ezio-melotti - -# Import (including importlib). -**/*import* @brettcannon @ericsnowcurrently @ncoghlan @warsaw -/Python/import.c @kumaraditya303 -Python/dynload_*.c @ericsnowcurrently -**/*freeze* @ericsnowcurrently -**/*frozen* @ericsnowcurrently -**/*modsupport* @ericsnowcurrently -**/*modulefinder* @ericsnowcurrently -**/*moduleobject* @ericsnowcurrently -**/*multiphase* @ericsnowcurrently -**/*pkgutil* @ericsnowcurrently -**/*pythonrun* @ericsnowcurrently -**/*runpy* @ericsnowcurrently -**/*singlephase* @ericsnowcurrently -Lib/test/test_module/ @ericsnowcurrently -Doc/c-api/module.rst @ericsnowcurrently -**/*importlib/resources/* @jaraco @warsaw @FFY00 -**/*importlib/metadata/* @jaraco @warsaw +# Argument Clinic +Tools/clinic/ @erlend-aasland @AA-Turner +Lib/test/test_clinic.py @erlend-aasland @AA-Turner +Doc/howto/clinic.rst @erlend-aasland @AA-Turner -# Dates and times -**/*datetime* @pganssle @abalkin -**/*str*time* @pganssle @abalkin -Doc/library/time.rst @pganssle @abalkin -Lib/test/test_time.py @pganssle @abalkin -Modules/timemodule.c @pganssle @abalkin -Python/pytime.c @pganssle @abalkin -Include/internal/pycore_time.h @pganssle @abalkin +# C Analyser +Tools/c-analyzer/ @ericsnowcurrently -# Email and related -**/*mail* @python/email-team -**/*smtp* @python/email-team -**/*mime* @python/email-team -**/*imap* @python/email-team -**/*poplib* @python/email-team +# C API Documentation Checks +Tools/check-c-api-docs/ @ZeroIntensity -# Exclude .mailmap from being owned by @python/email-team -/.mailmap +# Fuzzing +Modules/_xxtestfuzz/ @python/fuzzers +Lib/test/test_xxtestfuzz.py @python/fuzzers +.github/workflows/reusable-cifuzz.yml @python/fuzzers -# Garbage collector -/Modules/gcmodule.c @pablogsal -/Doc/library/gc.rst @pablogsal +# Limited C API & Stable ABI +Doc/c-api/stable.rst @encukou +Doc/data/*.abi @encukou +Misc/stable_abi.toml @encukou +Tools/build/stable_abi.py @encukou -# Parser -/Parser/ @pablogsal @lysnikolaou -/Tools/peg_generator/ @pablogsal @lysnikolaou -/Lib/test/test_peg_generator/ @pablogsal @lysnikolaou -/Grammar/python.gram @pablogsal @lysnikolaou -/Lib/tokenize.py @pablogsal @lysnikolaou -/Lib/test/test_tokenize.py @pablogsal @lysnikolaou +# SBOM +Misc/externals.spdx.json @sethmlarson +Misc/sbom.spdx.json @sethmlarson +Tools/build/generate_sbom.py @sethmlarson -# Code generator -/Tools/cases_generator/ @markshannon +# ABI check +Misc/libabigail.abignore @encukou -# AST -Python/ast.c @isidentical @JelleZijlstra @eclips4 @tomasr8 -Python/ast_preprocess.c @isidentical @eclips4 @tomasr8 -Parser/asdl.py @isidentical @JelleZijlstra @eclips4 @tomasr8 -Parser/asdl_c.py @isidentical @JelleZijlstra @eclips4 @tomasr8 -Lib/ast.py @isidentical @JelleZijlstra @eclips4 @tomasr8 -Lib/_ast_unparse.py @isidentical @JelleZijlstra @eclips4 @tomasr8 -Lib/test/test_ast/ @eclips4 @tomasr8 -# Mock -/Lib/unittest/mock.py @cjw296 -/Lib/test/test_unittest/testmock/* @cjw296 +# ---------------------------------------------------------------------------- +# Platform Support +# ---------------------------------------------------------------------------- + +# Android +Platforms/Android/ @mhsmith @freakboy3742 +Doc/using/android.rst @mhsmith @freakboy3742 +Lib/_android_support.py @mhsmith @freakboy3742 +Lib/test/test_android.py @mhsmith @freakboy3742 -# multiprocessing -**/*multiprocessing* @gpshead +# iOS +Doc/using/ios.rst @freakboy3742 +Lib/_ios_support.py @freakboy3742 +Platforms/Apple/ @freakboy3742 -# SQLite 3 -**/*sqlite* @berkerpeksag @erlend-aasland +# macOS +Mac/ @python/macos-team +Lib/_osx_support.py @python/macos-team +Lib/test/test__osx_support.py @python/macos-team -# subprocess -/Lib/subprocess.py @gpshead -/Lib/test/test_subprocess.py @gpshead -/Modules/*subprocess* @gpshead +# WebAssembly +Tools/wasm/README.md @brettcannon @freakboy3742 @emmatyping -# debugger -**/*pdb* @gaogaotiantian -**/*bdb* @gaogaotiantian +# WebAssembly (Emscripten) +Platforms/emscripten @freakboy3742 @emmatyping +Tools/wasm/emscripten @freakboy3742 @emmatyping -# Limited C API & stable ABI -Tools/build/stable_abi.py @encukou -Misc/stable_abi.toml @encukou -Doc/data/*.abi @encukou -Doc/c-api/stable.rst @encukou +# WebAssembly (WASI) +Platforms/WASI @brettcannon @emmatyping @savannahostrowski +Tools/wasm/wasi-env @brettcannon @emmatyping @savannahostrowski +Tools/wasm/wasi.py @brettcannon @emmatyping @savannahostrowski +Tools/wasm/wasi @brettcannon @emmatyping @savannahostrowski # Windows -/PC/ @python/windows-team -/PCbuild/ @python/windows-team - -# Urllib -**/*robotparser* @berkerpeksag +PC/ @python/windows-team +PCbuild/ @python/windows-team # Windows installer packages -/Tools/msi/ @python/windows-team -/Tools/nuget/ @python/windows-team +Tools/msi/ @python/windows-team +Tools/nuget/ @python/windows-team -# Misc -**/*itertools* @rhettinger -**/*collections* @rhettinger -**/*random* @rhettinger -**/*bisect* @rhettinger -**/*heapq* @rhettinger -**/*functools* @rhettinger +# Windows Launcher +PC/launcher.c @python/windows-team @vsajip -**/*dataclasses* @ericvsmith -**/*ensurepip* @pfmoore @pradyunsg +# ---------------------------------------------------------------------------- +# Interpreter Core +# ---------------------------------------------------------------------------- -/Doc/library/idle.rst @terryjreedy -**/*idlelib* @terryjreedy -**/*turtledemo* @terryjreedy +# AST +Lib/_ast_unparse.py @isidentical @JelleZijlstra @eclips4 @tomasr8 +Lib/ast.py @isidentical @JelleZijlstra @eclips4 @tomasr8 +Lib/test/test_ast/ @eclips4 @tomasr8 +Parser/asdl.py @isidentical @JelleZijlstra @eclips4 @tomasr8 +Parser/asdl_c.py @isidentical @JelleZijlstra @eclips4 @tomasr8 +Python/ast.c @isidentical @JelleZijlstra @eclips4 @tomasr8 +Python/ast_preprocess.c @isidentical @eclips4 @tomasr8 -**/*annotationlib* @JelleZijlstra -**/*typing* @JelleZijlstra @AlexWaygood +# Built-in types +Objects/call.c @markshannon +Objects/codeobject.c @markshannon +Objects/dict* @methane @markshannon +Objects/frameobject.c @markshannon +**/*genobject* @markshannon +Objects/object.c @ZeroIntensity +Objects/set* @rhettinger +Objects/type* @markshannon +Objects/typevarobject.c @JelleZijlstra +Objects/unionobject.c @JelleZijlstra -**/*ftplib @giampaolo -**/*shutil @giampaolo +# Byte code interpreter ('the eval loop') +Python/bytecodes.c @markshannon +Python/ceval* @markshannon +Tools/cases_generator/ @markshannon -**/*enum* @ethanfurman -**/*cgi* @ethanfurman -**/*tarfile* @ethanfurman +# Compiler (AST to byte code) +Python/assemble.c @markshannon @iritkatriel +Python/codegen.c @markshannon @iritkatriel +Python/compile.c @markshannon @iritkatriel +Python/flowgraph.c @markshannon @iritkatriel +Python/instruction_sequence.c @iritkatriel +Python/symtable.c @JelleZijlstra @carljm -**/*tomllib* @encukou @hauntsaninja +# Context variables & HAMT +**/contextvars* @1st1 +**/*hamt* @1st1 +Include/cpython/context.h @1st1 +Include/internal/pycore_context.h @1st1 +Lib/test/test_context.py @1st1 +Python/context.c @1st1 -**/*sysconfig* @FFY00 +# Core Modules +**/*bltinmodule* @ericsnowcurrently +**/*sysmodule* @ericsnowcurrently -**/*cjkcodecs* @corona10 +# Exceptions +Lib/test/test_except*.py @iritkatriel +Objects/exceptions.c @iritkatriel -# macOS -/Mac/ @python/macos-team -**/*osx_support* @python/macos-team +# Getpath +Lib/test/test_getpath.py @FFY00 +Modules/getpath* @FFY00 -# pathlib -**/*pathlib* @barneygale +# Hashing / ``hash()`` and related +Include/cpython/pyhash.h @gpshead @picnixz +Include/internal/pycore_pyhash.h @gpshead @picnixz +Include/pyhash.h @gpshead @picnixz +Python/pyhash.c @gpshead @picnixz -# zipfile.Path -**/*zipfile/_path/* @jaraco +# The import system (including importlib) +**/*import* @brettcannon @ericsnowcurrently @ncoghlan @warsaw @FFY00 +Python/import.c @brettcannon @ericsnowcurrently @ncoghlan @warsaw @FFY00 @kumaraditya303 +**/*freeze* @ericsnowcurrently +**/*frozen* @ericsnowcurrently +**/*modsupport* @ericsnowcurrently +**/*modulefinder* @ericsnowcurrently @FFY00 +**/*moduleobject* @ericsnowcurrently +**/*multiphase* @ericsnowcurrently +**/*pkgutil* @ericsnowcurrently @FFY00 +**/*pythonrun* @ericsnowcurrently +**/*runpy* @ericsnowcurrently @FFY00 +**/*singlephase* @ericsnowcurrently +Doc/c-api/module.rst @ericsnowcurrently +Lib/test/test_module/ @ericsnowcurrently +Python/dynload_*.c @ericsnowcurrently @FFY00 -# Argument Clinic -/Tools/clinic/** @erlend-aasland -/Lib/test/test_clinic.py @erlend-aasland -Doc/howto/clinic.rst @erlend-aasland - -# Subinterpreters -**/*interpreteridobject.* @ericsnowcurrently -**/*crossinterp* @ericsnowcurrently -Modules/_interp*module.c @ericsnowcurrently -Lib/test/test__interp*.py @ericsnowcurrently -Lib/concurrent/interpreters/ @ericsnowcurrently -Lib/test/support/channels.py @ericsnowcurrently -Doc/library/concurrent.interpreters.rst @ericsnowcurrently -Lib/test/test_interpreters/ @ericsnowcurrently -Lib/concurrent/futures/interpreter.py @ericsnowcurrently +# Initialisation +**/*initconfig* @ericsnowcurrently @FFY00 +**/*pathconfig* @ericsnowcurrently @FFY00 +**/*preconfig* @ericsnowcurrently @FFY00 +Doc/library/sys_path_init.rst @FFY00 +Doc/c-api/init_config.rst @FFY00 -# Android -**/*Android* @mhsmith @freakboy3742 -**/*android* @mhsmith @freakboy3742 +# Interpreter main program +Modules/main.c @ericsnowcurrently @FFY00 +Programs/_bootstrap_python.c @ericsnowcurrently @FFY00 +Programs/python.c @ericsnowcurrently @FFY00 + +# JIT +.github/workflows/jit.yml @savannahostrowski +Include/internal/pycore_jit.h @brandtbucher @savannahostrowski @diegorusso +Python/jit.c @brandtbucher @savannahostrowski @diegorusso +Tools/jit/ @brandtbucher @savannahostrowski @diegorusso +InternalDocs/jit.md @brandtbucher @savannahostrowski @diegorusso @AA-Turner + +# Lazy imports (PEP 810) +Objects/lazyimportobject.c @yhg1s @DinoV @pablogsal +Include/internal/pycore_lazyimportobject.h @yhg1s @DinoV @pablogsal +Lib/test/test_lazy_import @yhg1s @DinoV @pablogsal + +# Micro-op / μop / Tier 2 Optimiser +Python/optimizer.c @markshannon @Fidget-Spinner +Python/optimizer_analysis.c @markshannon @tomasr8 @Fidget-Spinner @savannahostrowski +Python/optimizer_bytecodes.c @markshannon @tomasr8 @Fidget-Spinner @savannahostrowski +Python/optimizer_symbols.c @markshannon @tomasr8 @Fidget-Spinner @savannahostrowski + +# Parser, Lexer, and Grammar +Grammar/python.gram @pablogsal @lysnikolaou +Lib/test/test_peg_generator/ @pablogsal @lysnikolaou +Lib/test/test_tokenize.py @pablogsal @lysnikolaou +Lib/tokenize.py @pablogsal @lysnikolaou +Parser/ @pablogsal @lysnikolaou +Tools/peg_generator/ @pablogsal @lysnikolaou + +# Runtime state/lifecycle +**/*gil* @ericsnowcurrently +**/*pylifecycle* @ericsnowcurrently @ZeroIntensity @FFY00 +**/*pystate* @ericsnowcurrently @ZeroIntensity @FFY00 +Include/internal/pycore_*_init.h @ericsnowcurrently +Include/internal/pycore_*_state.h @ericsnowcurrently +Include/internal/pycore_atexit.h @ericsnowcurrently +Include/internal/pycore_freelist.h @ericsnowcurrently +Include/internal/pycore_global_objects.h @ericsnowcurrently +Include/internal/pycore_interp.h @ericsnowcurrently +Include/internal/pycore_obmalloc.h @ericsnowcurrently +Include/internal/pycore_pymem.h @ericsnowcurrently +Include/internal/pycore_runtime.h @ericsnowcurrently +Include/internal/pycore_stackref.h @Fidget-Spinner +Include/internal/pycore_tstate.h @ericsnowcurrently +Tools/build/generate_global_objects.py @ericsnowcurrently + +# Remote Debugging +Python/remote_debug.h @pablogsal +Python/remote_debugging.c @pablogsal +Modules/_remote_debugging/ @pablogsal + +# Sub-Interpreters +**/*crossinterp* @ericsnowcurrently +**/*interpreteridobject.* @ericsnowcurrently +Doc/library/concurrent.interpreters.rst @ericsnowcurrently +Lib/concurrent/futures/interpreter.py @ericsnowcurrently +Lib/concurrent/interpreters/ @ericsnowcurrently +Lib/test/support/channels.py @ericsnowcurrently +Lib/test/test__interp*.py @ericsnowcurrently +Lib/test/test_interpreters/ @ericsnowcurrently +Modules/_interp*module.c @ericsnowcurrently + +# Template string literals (t-strings) +Lib/test/test_tstring.py @lysnikolaou +Objects/interpolationobject.c @lysnikolaou +Objects/templateobject.c @lysnikolaou + +# Tests +Lib/test/test_patma.py @brandtbucher +Lib/test/test_type_*.py @JelleZijlstra +Lib/test/test_capi/test_misc.py @markshannon -# iOS (but not termios) -**/iOS* @freakboy3742 -**/ios* @freakboy3742 -**/*_iOS* @freakboy3742 -**/*_ios* @freakboy3742 -**/*-iOS* @freakboy3742 -**/*-ios* @freakboy3742 -# WebAssembly -Tools/wasm/config.site-wasm32-emscripten @freakboy3742 -/Tools/wasm/README.md @brettcannon @freakboy3742 -/Tools/wasm/wasi-env @brettcannon -/Tools/wasm/wasi.py @brettcannon -/Tools/wasm/emscripten @freakboy3742 -/Tools/wasm/wasi @brettcannon +# ---------------------------------------------------------------------------- +# Standard Library +# ---------------------------------------------------------------------------- + +# Annotationlib +Doc/library/annotationlib.rst @JelleZijlstra +Lib/annotationlib.py @JelleZijlstra +Lib/test/test_annotationlib.py @JelleZijlstra + +# Argparse +Doc/**/argparse*.rst @savannahostrowski +Lib/argparse.py @savannahostrowski +Lib/test/test_argparse.py @savannahostrowski + +# Asyncio +Doc/library/asyncio*.rst @1st1 @asvetlov @kumaraditya303 @willingc +InternalDocs/asyncio.md @1st1 @asvetlov @kumaraditya303 @willingc @AA-Turner +Lib/asyncio/ @1st1 @asvetlov @kumaraditya303 @willingc +Lib/test/test_asyncio/ @1st1 @asvetlov @kumaraditya303 @willingc +Modules/_asynciomodule.c @1st1 @asvetlov @kumaraditya303 @willingc + +# Bisect +Doc/library/bisect.rst @rhettinger +Lib/bisect.py @rhettinger +Lib/test/test_bisect.py @rhettinger +Modules/_bisectmodule.c @rhettinger + +# Calendar +Lib/calendar.py @AA-Turner +Lib/test/test_calendar.py @AA-Turner + +# Cryptographic Primitives and Applications +**/*hashlib* @gpshead @picnixz +**/*hashopenssl* @gpshead @picnixz +**/*hmac* @gpshead @picnixz +**/*ssl* @gpshead @picnixz +Modules/_hacl/ @gpshead @picnixz +Modules/*blake* @gpshead @picnixz +Modules/*md5* @gpshead @picnixz +Modules/*sha* @gpshead @picnixz + +# Codecs +Modules/cjkcodecs/ @corona10 +Tools/unicode/gencjkcodecs.py @corona10 + +# Collections +Doc/library/collections.abc.rst @rhettinger +Doc/library/collections.rst @rhettinger +Lib/_collections_abc.py @rhettinger +Lib/collections/ @rhettinger +Lib/test/test_collections.py @rhettinger +Modules/_collectionsmodule.c @rhettinger -# SBOM -/Misc/externals.spdx.json @sethmlarson -/Misc/sbom.spdx.json @sethmlarson -/Tools/build/generate_sbom.py @sethmlarson +# Colorize +Lib/_colorize.py @hugovk +Lib/test/test__colorize.py @hugovk # Config Parser Lib/configparser.py @jaraco Lib/test/test_configparser.py @jaraco -# Doc sections -Doc/reference/ @willingc @AA-Turner +# Dataclasses +Doc/library/dataclasses.rst @ericvsmith +Lib/dataclasses.py @ericvsmith +Lib/test/test_dataclasses/ @ericvsmith +# Dates and times +Doc/**/*time.rst @pganssle @StanFromIreland +Doc/library/datetime-* @pganssle @StanFromIreland +Doc/library/zoneinfo.rst @pganssle @StanFromIreland +Include/datetime.h @pganssle @StanFromIreland +Include/internal/pycore_time.h @pganssle @StanFromIreland +Lib/test/test_zoneinfo/ @pganssle @StanFromIreland +Lib/zoneinfo/ @pganssle @StanFromIreland +Lib/*time.py @pganssle @StanFromIreland +Lib/test/datetimetester.py @pganssle @StanFromIreland +Lib/test/test_*time.py @pganssle @StanFromIreland +Modules/*zoneinfo* @pganssle @StanFromIreland +Modules/*time* @pganssle @StanFromIreland +Python/pytime.c @pganssle @StanFromIreland + +# Dbm +Doc/library/dbm.rst @corona10 @erlend-aasland @serhiy-storchaka +Lib/dbm/ @corona10 @erlend-aasland @serhiy-storchaka +Lib/test/test_dbm*.py @corona10 @erlend-aasland @serhiy-storchaka +Modules/*dbm* @corona10 @erlend-aasland @serhiy-storchaka + +# Email and related +**/*mail* @python/email-team +**/*smtp* @python/email-team +**/*mime* @python/email-team +**/*imap* @python/email-team +**/*poplib* @python/email-team + +# Ensurepip +Doc/library/ensurepip.rst @pfmoore @pradyunsg +Lib/ensurepip/ @pfmoore @pradyunsg +Lib/test/test_ensurepip.py @pfmoore @pradyunsg + +# Enum +Doc/howto/enum.rst @ethanfurman +Doc/library/enum.rst @ethanfurman +Lib/enum.py @ethanfurman +Lib/test/test_enum.py @ethanfurman +Lib/test/test_json/test_enum.py @ethanfurman + +# FTP +Doc/library/ftplib.rst @giampaolo +Lib/ftplib.py @giampaolo +Lib/test/test_ftplib.py @giampaolo + +# Functools +Doc/library/functools.rst @rhettinger +Lib/functools.py @rhettinger +Lib/test/test_functools.py @rhettinger +Modules/_functoolsmodule.c @rhettinger + +# Garbage collector +Modules/gcmodule.c @pablogsal +Doc/library/gc.rst @pablogsal +InternalDocs/garbage_collector.md @pablogsal + +# Gettext +Doc/library/gettext.rst @tomasr8 +Lib/gettext.py @tomasr8 +Lib/test/test_gettext.py @tomasr8 +Tools/i18n/pygettext.py @tomasr8 + +# Heapq +Doc/library/heapq* @rhettinger +Lib/heapq.py @rhettinger +Lib/test/test_heapq.py @rhettinger +Modules/_heapqmodule.c @rhettinger + +# HTML +Doc/library/html* @ezio-melotti +Lib/html/ @ezio-melotti +Lib/_markupbase.py @ezio-melotti +Lib/test/test_html*.py @ezio-melotti +Tools/build/parse_html5_entities.py @ezio-melotti + +# IDLE +Doc/library/idle.rst @terryjreedy +Lib/idlelib/ @terryjreedy +Lib/turtledemo/ @terryjreedy + +# importlib.metadata +Doc/library/importlib.metadata.rst @jaraco @warsaw @FFY00 +Lib/importlib/metadata/ @jaraco @warsaw @FFY00 +Lib/test/test_importlib/metadata/ @jaraco @warsaw @FFY00 + +# importlib.resources +Doc/library/importlib.resources.abc.rst @jaraco @warsaw @FFY00 +Doc/library/importlib.resources.rst @jaraco @warsaw @FFY00 +Lib/importlib/resources/ @jaraco @warsaw @FFY00 +Lib/test/test_importlib/resources/ @jaraco @warsaw @FFY00 + +# Itertools +Doc/library/itertools.rst @rhettinger +Lib/test/test_itertools.py @rhettinger +Modules/itertoolsmodule.c @rhettinger + +# Logging +Doc/**/logging* @vsajip +Lib/logging/ @vsajip +Lib/test/test_logging.py @vsajip + +# Multiprocessing +Doc/library/multiprocessing*.rst @gpshead +Lib/multiprocessing/ @gpshead +Lib/test/*multiprocessing.py @gpshead +Lib/test/test_multiprocessing*/ @gpshead +Modules/_multiprocessing/ @gpshead + +# Pathlib +Doc/library/pathlib.rst @barneygale +Lib/pathlib/ @barneygale +Lib/test/test_pathlib/ @barneygale + +# Pdb & Bdb +Doc/library/bdb.rst @gaogaotiantian +Doc/library/pdb.rst @gaogaotiantian +Lib/bdb.py @gaogaotiantian +Lib/pdb.py @gaogaotiantian +Lib/test/test_bdb.py @gaogaotiantian +Lib/test/test_pdb.py @gaogaotiantian +Lib/test/test_remote_pdb.py @gaogaotiantian + +# Pydoc +Lib/pydoc.py @AA-Turner +Lib/pydoc_data/ @AA-Turner +Lib/test/test_pydoc/ @AA-Turner + +# Profiling (Sampling) +Doc/library/profiling*.rst @pablogsal +Lib/profiling/ @pablogsal +Lib/test/test_profiling/ @pablogsal + +# PyREPL +Lib/_pyrepl/ @pablogsal @lysnikolaou @ambv +Lib/test/test_pyrepl/ @pablogsal @lysnikolaou @ambv + +# Random +Doc/library/random.rst @rhettinger +Lib/random.py @rhettinger +Lib/test/test_random.py @rhettinger +Modules/_randommodule.c @rhettinger + +# Shutil +Doc/library/shutil.rst @giampaolo +Lib/shutil.py @giampaolo +Lib/test/test_shutil.py @giampaolo + +# Site +Lib/site.py @FFY00 @warsaw +Lib/test/test_site.py @FFY00 @warsaw +Doc/library/site.rst @FFY00 @warsaw + +# string.templatelib +Doc/library/string.templatelib.rst @lysnikolaou @AA-Turner +Lib/string/templatelib.py @lysnikolaou @AA-Turner +Lib/test/test_string/test_templatelib.py @lysnikolaou @AA-Turner + +# Sysconfig +**/*sysconfig* @FFY00 + +# SQLite 3 +Doc/library/sqlite3.rst @erlend-aasland +Lib/sqlite3/ @erlend-aasland +Lib/test/test_sqlite3/ @erlend-aasland +Modules/_sqlite/ @erlend-aasland + +# Subprocess +Lib/subprocess.py @gpshead +Lib/test/test_subprocess.py @gpshead +Modules/*subprocess* @gpshead + +# Tarfile +Doc/library/tarfile.rst @ethanfurman +Lib/tarfile.py @ethanfurman +Lib/test/test_tarfile.py @ethanfurman + +# TOML +Doc/library/tomllib.rst @encukou @hauntsaninja +Lib/test/test_tomllib/ @encukou @hauntsaninja +Lib/tomllib/ @encukou @hauntsaninja + +# Typing +Doc/library/typing.rst @JelleZijlstra @AlexWaygood +Lib/test/test_typing.py @JelleZijlstra @AlexWaygood +Lib/test/typinganndata/ @JelleZijlstra @AlexWaygood +Lib/typing.py @JelleZijlstra @AlexWaygood +Modules/_typingmodule.c @JelleZijlstra @AlexWaygood + +# Types +Lib/test/test_types.py @AA-Turner +Lib/types.py @AA-Turner +Modules/_typesmodule.c @AA-Turner + +# Unittest +Lib/unittest/mock.py @cjw296 +Lib/test/test_unittest/testmock/ @cjw296 + +# Venv +**/*venv* @vsajip @FFY00 + +# Weakref **/*weakref* @kumaraditya303 -# Colorize -Lib/_colorize.py @hugovk -Lib/test/test__colorize.py @hugovk +# Zipfile.Path +Lib/test/test_zipfile/_path/ @jaraco +Lib/zipfile/_path/ @jaraco -# Fuzzing -Modules/_xxtestfuzz/ @ammaraskar +# Zstandard +Lib/compression/zstd/ @AA-Turner @emmatyping +Lib/test/test_zstd.py @AA-Turner @emmatyping +Modules/_zstd/ @AA-Turner @emmatyping -# t-strings -**/*interpolationobject* @lysnikolaou -**/*templateobject* @lysnikolaou -**/*templatelib* @lysnikolaou -**/*tstring* @lysnikolaou +# ---------------------------------------------------------------------------- -# Remote debugging -Python/remote_debug.h @pablogsal -Python/remote_debugging.c @pablogsal -Modules/_remote_debugging_module.c @pablogsal @ambv @1st1 +# Exclusions from .github/CODEOWNERS should go at the very end +# because the final matching pattern will take precedence. + +# Exclude .mailmap from being owned by @python/email-team +.mailmap -# gettext -**/*gettext* @tomasr8 +# Exclude Argument Clinic directories +Modules/**/clinic/ +Objects/**/clinic/ +PC/**/clinic/ +Python/**/clinic/ diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index 2ef9cdc191481e5..94b67ce3dbe3414 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -4,7 +4,7 @@ Contributing to Python Build Status ------------ -- `Buildbot status overview `_ +- `Buildbot status overview `_ - `GitHub Actions status `_ @@ -28,23 +28,23 @@ Please be aware that our workflow does deviate slightly from the typical GitHub project. Details on how to properly submit a pull request are covered in `Lifecycle of a Pull Request `_. We utilize various bots and status checks to help with this, so do follow the -comments they leave and their "Details" links, respectively. The key points of -our workflow that are not covered by a bot or status check are: +comments they leave and their "Details" links, respectively. -- All discussions that are not directly related to the code in the pull request - should happen on `GitHub Issues `_. -- Upon your first non-trivial pull request (which includes documentation changes), - feel free to add yourself to ``Misc/ACKS`` +The final key part of our workflow is that all discussions that are not +directly related to the code in the pull request should happen on +`GitHub Issues `__, generally in the +pull request's parent issue. Setting Expectations -------------------- -Due to the fact that this project is entirely volunteer-run (i.e. no one is paid -to work on Python full-time), we unfortunately can make no guarantees as to if +Due to the fact that this project is run by volunteers, +unfortunately we cannot make any guarantees as to if or when a core developer will get around to reviewing your pull request. If no core developer has done a review or responded to changes made because of a -"changes requested" review, please feel free to email python-dev to ask if -someone could take a look at your pull request. +"changes requested" review within a month, you can ask for someone to +review your pull request via a post in the `Core Development Discourse +category `__. Code of Conduct diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index da70710b7ecfa39..177615621f6b8c7 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -34,13 +34,13 @@ body: label: "CPython versions tested on:" multiple: true options: - - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" - "3.14" - "3.15" + - "3.16" - "CPython main branch" validations: required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 75d174307ce1601..de6e8756b03d800 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -5,3 +5,6 @@ contact_links: - name: "Proposing new features" about: "Submit major feature proposal (e.g. syntax changes) to an ideas forum first." url: "https://discuss.python.org/c/ideas/6" + - name: "Python Install Manager issues" + about: "Report issues with the Python Install Manager (for Windows)" + url: "https://github.com/python/pymanager/issues" diff --git a/.github/ISSUE_TEMPLATE/crash.yml b/.github/ISSUE_TEMPLATE/crash.yml index 470ad581367b103..81ae91e5b0af973 100644 --- a/.github/ISSUE_TEMPLATE/crash.yml +++ b/.github/ISSUE_TEMPLATE/crash.yml @@ -27,13 +27,13 @@ body: label: "CPython versions tested on:" multiple: true options: - - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" - "3.14" - "3.15" + - "3.16" - "CPython main branch" validations: required: true diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md deleted file mode 100644 index 174fd39171d47dd..000000000000000 --- a/.github/ISSUE_TEMPLATE/documentation.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -name: Documentation -about: Report a problem with the documentation -labels: "docs" ---- - -# Documentation - -(A clear and concise description of the issue.) diff --git a/.github/ISSUE_TEMPLATE/documentation.yml b/.github/ISSUE_TEMPLATE/documentation.yml new file mode 100644 index 000000000000000..d720cf9c4de91e2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.yml @@ -0,0 +1,16 @@ +name: Documentation +description: Report a problem with the documentation +labels: ["docs"] +body: + - type: markdown + attributes: + value: | + > [!NOTE] + > Trivial changes (for example typos) don’t require an issue before opening a PR. + - type: textarea + id: description + attributes: + label: "Documentation" + description: "A clear and concise description of the issue. Include a link to the page." + validations: + required: true diff --git a/.github/SECURITY.md b/.github/SECURITY.md index 923720bce0bc3bf..5bf4b40947bea70 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -1,17 +1,24 @@ # Security Policy -## Supported Versions +Python [provides a security policy and threat model](https://devguide.python.org/security/policy/) +in the Python Developer's Guide documenting what bugs are vulnerabilities, +how to structure reports, and what versions of Python accept reports. -The Python team applies security fixes according to the table -in [the devguide]( -https://devguide.python.org/versions/#supported-versions -). +Python Security Response Team (PSRT) members +balance security work against many other responsibilities. Please be thoughtful +about the time and attention your report requires. Repeated failure to respect +the security policy will result in future reports being rejected, or the +reporter being banned from the ``python`` GitHub organization, regardless of +technical merit. ## Reporting a Vulnerability -Please read the guidelines on reporting security issues [on the -official website](https://www.python.org/dev/security/) for -instructions on how to report a security-related problem to -the Python team responsibly. +The [Python security policy](https://devguide.python.org/security/policy/) +documents [how to submit a vulnerability report](https://devguide.python.org/security/policy/#how-to-submit-a-vulnerability-report) +using GitHub Security Advisories. Please read the security policy +prior to filing a vulnerability report, especially the section on [what information to +include and exclude](https://devguide.python.org/security/policy/#what-to-include-and-how-to-structure-a-vulnerability-report) +in vulnerability reports. Following the security policy means the PSRT can +quickly and efficiently triage your report, not following the security policy +will only delay triaging your report. -To reach the response team, email `security at python dot org`. diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 68aae1963574141..eacfff24889021b 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,7 +1,3 @@ -self-hosted-runner: - # Pending https://github.com/rhysd/actionlint/issues/533 - labels: ["windows-11-arm"] - config-variables: null paths: diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c8a3165d6903645..4b77646e22db4ba 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,7 +3,7 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "monthly" + interval: "quarterly" labels: - "skip issue" - "skip news" @@ -12,10 +12,21 @@ updates: update-types: - "version-update:semver-minor" - "version-update:semver-patch" + groups: + actions: + patterns: + - "*" + cooldown: + # https://blog.yossarian.net/2025/11/21/We-should-all-be-using-dependency-cooldowns + # Cooldowns protect against supply chain attacks by avoiding the + # highest-risk window immediately after new releases. + default-days: 14 - package-ecosystem: "pip" directory: "/Tools/" schedule: - interval: "monthly" + interval: "quarterly" labels: - "skip issue" - "skip news" + cooldown: + default-days: 14 diff --git a/.github/workflows/add-issue-header.yml b/.github/workflows/add-issue-header.yml index 3cbc23af578d106..4c25976b9c24f72 100644 --- a/.github/workflows/add-issue-header.yml +++ b/.github/workflows/add-issue-header.yml @@ -12,6 +12,8 @@ on: # Only ever run once - opened +permissions: + contents: read jobs: add-header: @@ -20,7 +22,7 @@ jobs: issues: write timeout-minutes: 5 steps: - - uses: actions/github-script@v7 + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: # language=JavaScript script: | diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f07f5e8040acf02..43f8e0c010ed1bc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,7 +64,7 @@ jobs: run: | apt update && apt install git -yq git config --global --add safe.directory "$GITHUB_WORKSPACE" - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 1 persist-credentials: false @@ -101,28 +101,16 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.x' - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}-${{ env.pythonLocation }} - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Add ccache to PATH - run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Configure CPython run: | # Build Python with the libpython dynamic library @@ -130,7 +118,7 @@ jobs: - name: Build CPython run: | make -j4 regen-all - make regen-stdlib-module-names regen-sbom regen-unicodedata + make regen-stdlib-module-names regen-sbom - name: Check for changes run: | git add -u @@ -153,6 +141,14 @@ jobs: if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME run: make check-c-globals + check-c-api-docs: + name: C API Docs + needs: build-context + if: >- + needs.build-context.outputs.run-tests == 'true' + || needs.build-context.outputs.run-docs == 'true' + uses: ./.github/workflows/reusable-check-c-api-docs.yml + build-windows: name: >- Windows @@ -169,17 +165,25 @@ jobs: free-threading: - false - true + interpreter: + - switch-case exclude: # Skip Win32 on free-threaded builds - { arch: Win32, free-threading: true } + include: + # msvc::musttail is currently only supported on x64, + # and only supported on 3.15+. + - { arch: x64, free-threading: false, interpreter: tail-call } + - { arch: x64, free-threading: true, interpreter: tail-call } uses: ./.github/workflows/reusable-windows.yml with: arch: ${{ matrix.arch }} free-threading: ${{ matrix.free-threading }} + interpreter: ${{ matrix.interpreter }} build-windows-msi: - name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category - Windows MSI${{ '' }} + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: Windows MSI${{ '' }} # zizmor: ignore[obfuscation] needs: build-context if: fromJSON(needs.build-context.outputs.run-windows-msi) strategy: @@ -198,32 +202,23 @@ jobs: macOS ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-macos == 'true' strategy: fail-fast: false matrix: - # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. - # macOS 13 only runs tests against the GIL-enabled CPython. - # Cirrus used for upstream, macos-14 for forks. + # macos-26 is Apple Silicon, macos-26-intel is Intel. + # macos-26-intel only runs tests against the GIL-enabled CPython. os: - - ghcr.io/cirruslabs/macos-runner:sonoma - - macos-14 - - macos-13 - is-fork: # only used for the exclusion trick - - ${{ github.repository_owner != 'python' }} + - macos-26 + - macos-26-intel free-threading: - false - true exclude: - - os: ghcr.io/cirruslabs/macos-runner:sonoma - is-fork: true - - os: macos-14 - is-fork: false - - os: macos-13 + - os: macos-26-intel free-threading: true uses: ./.github/workflows/reusable-macos.yml with: - config_hash: ${{ needs.build-context.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} @@ -233,7 +228,7 @@ jobs: ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} ${{ fromJSON(matrix.bolt) && '(bolt)' || '' }} needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: fail-fast: false matrix: @@ -253,169 +248,160 @@ jobs: # BOLT currently crashes during instrumentation on aarch64 - os: ubuntu-24.04-arm bolt: true + include: + # Enable CPU-intensive tests on ARM (default build only) + - os: ubuntu-24.04-arm + bolt: false + free-threading: false + test-opts: '-u cpu' uses: ./.github/workflows/reusable-ubuntu.yml with: - config_hash: ${{ needs.build-context.outputs.config-hash }} bolt-optimizations: ${{ matrix.bolt }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} + test-opts: ${{ matrix.test-opts || '' }} - build-ubuntu-ssltests-openssl: - name: 'Ubuntu SSL tests with OpenSSL' - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04] - openssl_ver: [3.0.16, 3.1.8, 3.2.4, 3.3.3, 3.4.1] - # See Tools/ssl/make_ssl_data.py for notes on adding a new version - env: - OPENSSL_VER: ${{ matrix.openssl_ver }} - MULTISSL_DIR: ${{ github.workspace }}/multissl - OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }} - LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - - name: Configure CPython - run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR" - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: SSL tests - run: ./python Lib/test/ssltests.py - - build-ubuntu-ssltests-awslc: - name: 'Ubuntu SSL tests with AWS-LC' + build-ubuntu-ssltests: + name: 'Ubuntu SSL tests' runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: fail-fast: false matrix: os: [ubuntu-24.04] - awslc_ver: [1.55.0] + ssllib: + # See Tools/ssl/make_ssl_data.py for notes on adding a new version + ## OpenSSL + # Keep 1.1.1w in our list despite it being upstream EOL and otherwise + # unsupported as it most resembles other 1.1.1-work-a-like ssl APIs + # supported by important vendors such as AWS-LC. + - { name: openssl, version: 1.1.1w } + - { name: openssl, version: 3.0.21 } + - { name: openssl, version: 3.4.6 } + - { name: openssl, version: 3.5.7 } + - { name: openssl, version: 3.6.3 } + - { name: openssl, version: 4.0.1 } + ## AWS-LC + - { name: aws-lc, version: 5.0.0 } env: - AWSLC_VER: ${{ matrix.awslc_ver}} + SSLLIB_VER: ${{ matrix.ssllib.version }} MULTISSL_DIR: ${{ github.workspace }}/multissl - OPENSSL_DIR: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }} - LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}/lib + SSLLIB_DIR: ${{ github.workspace }}/multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }} + LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }}/lib steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure SSL lib env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore AWS-LC build' - id: cache-aws-lc - uses: actions/cache@v4 + - name: 'Restore SSL library build' + id: cache-ssl-lib + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: - path: ./multissl/aws-lc/${{ matrix.awslc_ver }} - key: ${{ matrix.os }}-multissl-aws-lc-${{ matrix.awslc_ver }} - - name: Install AWS-LC - if: steps.cache-aws-lc.outputs.cache-hit != 'true' + path: ./multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }} + key: ${{ matrix.os }}-multissl-${{ matrix.ssllib.name }}-${{ matrix.ssllib.version }} + - name: Install SSL Library + if: steps.cache-ssl-lib.outputs.cache-hit != 'true' run: | python3 Tools/ssl/multissltests.py \ --steps=library \ --base-directory "$MULTISSL_DIR" \ - --awslc ${{ matrix.awslc_ver }} \ + '--${{ matrix.ssllib.name }}' '${{ matrix.ssllib.version }}' \ --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Configure CPython run: | ./configure CFLAGS="-fdiagnostics-format=json" \ --config-cache \ --enable-slower-safety \ --with-pydebug \ - --with-openssl="$OPENSSL_DIR" \ + --with-openssl="$SSLLIB_DIR" \ --with-builtin-hashlib-hashes=blake2 \ --with-ssl-default-suites=openssl - name: Build CPython - run: make -j + run: make -j4 - name: Display build info run: make pythoninfo - - name: Verify python is linked to AWS-LC - run: ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC + - name: Verify python is linked to the right lib + run: | + ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' \ + | grep -iE '${{ matrix.ssllib.name }}.*${{ matrix.ssllib.version }}' - name: SSL tests run: ./python Lib/test/ssltests.py + build-android: + name: Android (${{ matrix.arch }}) + needs: build-context + if: needs.build-context.outputs.run-android == 'true' + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - arch: aarch64 + runs-on: macos-26 + - arch: x86_64 + runs-on: ubuntu-24.04 + + runs-on: ${{ matrix.runs-on }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Build and test + run: python3 Platforms/Android ci --fast-ci ${{ matrix.arch }}-linux-android + + build-ios: + name: iOS + needs: build-context + if: needs.build-context.outputs.run-ios == 'true' + timeout-minutes: 60 + runs-on: macos-14 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + # GitHub recommends explicitly selecting the desired Xcode version: + # https://github.com/actions/runner-images/issues/12541#issuecomment-3083850140 + # This became a necessity as a result of + # https://github.com/actions/runner-images/issues/12541 and + # https://github.com/actions/runner-images/issues/12751. + - name: Select Xcode version + run: | + sudo xcode-select --switch /Applications/Xcode_15.4.app + + - name: Build and test + run: python3 Platforms/Apple ci iOS --fast-ci --simulator 'iPhone SE (3rd generation),OS=17.5' + + build-emscripten: + name: 'Emscripten' + needs: build-context + if: needs.build-context.outputs.run-emscripten == 'true' + uses: ./.github/workflows/reusable-emscripten.yml + build-wasi: name: 'WASI' needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-wasi == 'true' uses: ./.github/workflows/reusable-wasi.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} test-hypothesis: name: "Hypothesis tests on Ubuntu" runs-on: ubuntu-24.04 timeout-minutes: 60 needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' env: - OPENSSL_VER: 3.0.16 + OPENSSL_VER: 3.5.7 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Register gcc problem matcher @@ -429,20 +415,13 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" @@ -453,11 +432,6 @@ jobs: run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -488,7 +462,7 @@ jobs: ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" - name: 'Restore Hypothesis database' id: cache-hypothesis-database - uses: actions/cache@v4 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ key: hypothesis-database-${{ github.head_ref || github.run_id }} @@ -515,7 +489,7 @@ jobs: -x test_subprocess \ -x test_signal \ -x test_sysconfig - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 if: always() with: name: hypothesis-example-db @@ -526,32 +500,27 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: fail-fast: false matrix: os: [ubuntu-24.04] env: - OPENSSL_VER: 3.0.16 + OPENSSL_VER: 3.5.7 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Set up GCC-10 for ASAN - uses: egor-tensin/setup-gcc@v1 + uses: egor-tensin/setup-gcc@a2861a8b8538f49cf2850980acccf6b05a1b2ae4 # v2.0 with: version: 10 - name: Configure OpenSSL env vars @@ -561,23 +530,15 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Configure CPython - run: ./configure --config-cache --with-address-sanitizer --without-pymalloc + run: ./configure --config-cache --with-address-sanitizer --without-pymalloc --with-openssl="$OPENSSL_DIR" - name: Build CPython run: make -j4 - name: Display build info @@ -586,10 +547,10 @@ jobs: run: xvfb-run make ci build-san: - name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category - Sanitizers${{ '' }} + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: Sanitizers${{ '' }} # zizmor: ignore[obfuscation] needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: fail-fast: false matrix: @@ -607,7 +568,6 @@ jobs: uses: ./.github/workflows/reusable-san.yml with: sanitizer: ${{ matrix.sanitizer }} - config_hash: ${{ needs.build-context.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} cross-build-linux: @@ -615,18 +575,13 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Set build dir @@ -650,45 +605,49 @@ jobs: run: | "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed - # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: - name: CIFuzz - runs-on: ubuntu-latest - timeout-minutes: 60 + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: CIFuzz${{ '' }} # zizmor: ignore[obfuscation] needs: build-context - if: needs.build-context.outputs.run-ci-fuzz == 'true' + if: >- + needs.build-context.outputs.run-ci-fuzz == 'true' + || needs.build-context.outputs.run-ci-fuzz-stdlib == 'true' permissions: + contents: read security-events: write strategy: fail-fast: false matrix: - sanitizer: [address, undefined, memory] - steps: - - name: Build fuzzers (${{ matrix.sanitizer }}) - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master - with: + sanitizer: + - address + oss-fuzz-project-name: + - cpython3 + - python3-libraries + include: + - sanitizer: undefined oss-fuzz-project-name: cpython3 - sanitizer: ${{ matrix.sanitizer }} - - name: Run fuzzers (${{ matrix.sanitizer }}) - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master - with: - fuzz-seconds: 600 + - sanitizer: memory oss-fuzz-project-name: cpython3 - output-sarif: true - sanitizer: ${{ matrix.sanitizer }} - - name: Upload crash - if: failure() && steps.build.outcome == 'success' - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.sanitizer }}-artifacts - path: ./out/artifacts - - name: Upload SARIF - if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: cifuzz-sarif/results.sarif - checkout_path: cifuzz-sarif + exclude: + # Note that the 'no-exclude' sentinel below is to prevent + # an empty string value from excluding all jobs and causing + # GHA to create a 'default' matrix entry with all empty values. + - oss-fuzz-project-name: >- + ${{ + needs.build-context.outputs.run-ci-fuzz == 'true' + && 'no-exclude' + || 'cpython3' + }} + - oss-fuzz-project-name: >- + ${{ + needs.build-context.outputs.run-ci-fuzz-stdlib == 'true' + && 'no-exclude' + || 'python3-libraries' + }} + uses: ./.github/workflows/reusable-cifuzz.yml + with: + oss-fuzz-project-name: ${{ matrix.oss-fuzz-project-name }} + sanitizer: ${{ matrix.sanitizer }} all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass @@ -699,12 +658,14 @@ jobs: - check-docs - check-autoconf-regen - check-generated-files + - check-c-api-docs - build-windows - build-windows-msi - build-macos - build-ubuntu - - build-ubuntu-ssltests-awslc - - build-ubuntu-ssltests-openssl + - build-ubuntu-ssltests + - build-ios + - build-emscripten - build-wasi - test-hypothesis - build-asan @@ -718,48 +679,50 @@ jobs: uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe with: allowed-failures: >- + build-android, + build-emscripten, build-windows-msi, - build-ubuntu-ssltests-awslc, - build-ubuntu-ssltests-openssl, + build-ubuntu-ssltests, test-hypothesis, cifuzz, allowed-skips: >- - ${{ - !fromJSON(needs.build-context.outputs.run-docs) - && ' - check-docs, - ' - || '' - }} + ${{ !fromJSON(needs.build-context.outputs.run-docs) && 'check-docs,' || '' }} ${{ needs.build-context.outputs.run-tests != 'true' && ' check-autoconf-regen, check-generated-files, - build-macos, - build-ubuntu, - build-ubuntu-ssltests-awslc, - build-ubuntu-ssltests-openssl, - build-wasi, - test-hypothesis, - build-asan, - build-san, - cross-build-linux, ' || '' }} ${{ - !fromJSON(needs.build-context.outputs.run-windows-tests) - && ' - build-windows, - ' + !fromJSON(needs.build-context.outputs.run-tests) + && !fromJSON(needs.build-context.outputs.run-docs) + && 'check-c-api-docs,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-windows-tests) && 'build-windows,' || '' }} ${{ !fromJSON(needs.build-context.outputs.run-ci-fuzz) + && !fromJSON(needs.build-context.outputs.run-ci-fuzz-stdlib) + && 'cifuzz,' || + '' + }} + ${{ !fromJSON(needs.build-context.outputs.run-macos) && 'build-macos,' || '' }} + ${{ + !fromJSON(needs.build-context.outputs.run-ubuntu) && ' - cifuzz, + build-ubuntu, + build-ubuntu-ssltests, + test-hypothesis, + build-asan, + build-san, + cross-build-linux, ' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-android) && 'build-android,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-ios) && 'build-ios,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-emscripten) && 'build-emscripten,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-wasi) && 'build-wasi,' || '' }} jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml deleted file mode 100644 index a09a30587b35ebb..000000000000000 --- a/.github/workflows/documentation-links.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Read the Docs PR preview -# Automatically edits a pull request's descriptions with a link -# to the documentation's preview on Read the Docs. - -on: - pull_request_target: - types: - - opened - paths: - - 'Doc/**' - - '.github/workflows/doc.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - documentation-links: - runs-on: ubuntu-latest - permissions: - pull-requests: write - timeout-minutes: 5 - - steps: - - uses: readthedocs/actions/preview@v1 - with: - project-slug: "cpython-previews" - single-version: "true" diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 947badff816c099..66bff36fe5e90c0 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -1,25 +1,18 @@ name: JIT on: pull_request: - paths: + paths: &paths - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' - 'Python/executor_cases.c.h' - 'Python/optimizer_cases.c.h' + - '**_testinternalcapi**' - '!Python/perf_jit_trampoline.c' - '!**/*.md' - '!**/*.ini' push: - paths: - - '**jit**' - - 'Python/bytecodes.c' - - 'Python/optimizer*.c' - - 'Python/executor_cases.c.h' - - 'Python/optimizer_cases.c.h' - - '!Python/perf_jit_trampoline.c' - - '!**/*.md' - - '!**/*.ini' + paths: *paths workflow_dispatch: permissions: @@ -31,16 +24,20 @@ concurrency: env: FORCE_COLOR: 1 + LLVM_VERSION: 21 jobs: interpreter: name: Interpreter (Debug) runs-on: ubuntu-24.04 - timeout-minutes: 90 + timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh - name: Build tier two interpreter run: | ./configure --enable-experimental-jit=interpreter --with-pydebug @@ -48,11 +45,12 @@ jobs: - name: Test tier two interpreter run: | ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - jit: + + windows: name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) - needs: interpreter + runs-on: ${{ matrix.runner }} - timeout-minutes: 90 + timeout-minutes: 60 strategy: fail-fast: false matrix: @@ -60,100 +58,159 @@ jobs: - i686-pc-windows-msvc/msvc - x86_64-pc-windows-msvc/msvc - aarch64-pc-windows-msvc/msvc - - x86_64-apple-darwin/clang - - aarch64-apple-darwin/clang - - x86_64-unknown-linux-gnu/gcc - - aarch64-unknown-linux-gnu/gcc debug: - true - false - llvm: - - 19 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 - runner: windows-latest + runner: windows-2025-vs2026 - target: x86_64-pc-windows-msvc/msvc architecture: x64 - runner: windows-latest + runner: windows-2025-vs2026 - target: aarch64-pc-windows-msvc/msvc architecture: ARM64 runner: windows-11-arm - - target: x86_64-apple-darwin/clang - architecture: x86_64 - runner: macos-13 - - target: aarch64-apple-darwin/clang - architecture: aarch64 - runner: macos-14 - - target: x86_64-unknown-linux-gnu/gcc - architecture: x86_64 - runner: ubuntu-24.04 - - target: aarch64-unknown-linux-gnu/gcc - architecture: aarch64 - runner: ubuntu-24.04-arm steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' - # PCbuild downloads LLVM automatically: - - name: Windows - if: runner.os == 'Windows' + - name: Build run: | ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} + - name: Test + run: | ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. - # This is a bug in the macOS runner image where the pre-installed Python is installed in the same - # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes - # the symlink to the pre-installed Python so that the Homebrew Python is used instead. - - name: macOS - if: runner.os == 'macOS' + macos: + name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) + + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + target: + - x86_64-apple-darwin/clang + - aarch64-apple-darwin/clang + debug: + - true + - false + include: + - target: x86_64-apple-darwin/clang + runner: macos-15-intel + - target: aarch64-apple-darwin/clang + runner: macos-15 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - name: Install LLVM run: | brew update - find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete - brew install llvm@${{ matrix.llvm }} + brew install llvm@${{ env.LLVM_VERSION }} + - name: Build + run: | export SDKROOT="$(xcrun --show-sdk-path)" + # Set MACOSX_DEPLOYMENT_TARGET and -Werror=unguarded-availability to + # make sure we don't break downstream distributors (like uv): + export CFLAGS_JIT='-Werror=unguarded-availability' + export MACOSX_DEPLOYMENT_TARGET=10.15 ./configure --enable-experimental-jit --enable-universalsdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 + - name: Test + run: | ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: Linux - if: runner.os == 'Linux' + linux: + name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) + + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + target: + - x86_64-unknown-linux-gnu/gcc + - aarch64-unknown-linux-gnu/gcc + debug: + - true + - false + include: + - target: x86_64-unknown-linux-gnu/gcc + runner: ubuntu-24.04 + - target: aarch64-unknown-linux-gnu/gcc + runner: ubuntu-24.04-arm + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + - name: Build run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} + export PATH="$(llvm-config-${{ env.LLVM_VERSION }} --bindir):$PATH" ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 + - name: Test + run: | ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - # XXX: GH-133171 - # jit-with-disabled-gil: - # name: Free-Threaded (Debug) - # needs: interpreter - # runs-on: ubuntu-24.04 - # timeout-minutes: 90 - # strategy: - # fail-fast: false - # matrix: - # llvm: - # - 19 - # steps: - # - uses: actions/checkout@v4 - # with: - # persist-credentials: false - # - uses: actions/setup-python@v5 - # with: - # python-version: '3.11' - # - name: Build with JIT enabled and GIL disabled - # run: | - # sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - # export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - # ./configure --enable-experimental-jit --with-pydebug --disable-gil - # make all --jobs 4 - # - name: Run tests - # run: | - # ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + linux-extras: + name: ${{ matrix.name }} + + runs-on: ubuntu-24.04 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - name: Free-Threaded (Debug) + configure_flags: --enable-experimental-jit --with-pydebug --disable-gil + continue_on_error: true + - name: JIT without optimizations (Debug) + configure_flags: --enable-experimental-jit --with-pydebug + test_env: "PYTHON_UOPS_OPTIMIZE=0" + - name: JIT with stress testing (Debug) + configure_flags: --enable-experimental-jit --with-pydebug + test_env: "PYTHON_JIT_STRESS=1" + - name: JIT with tail calling interpreter + configure_flags: --enable-experimental-jit --with-tail-call-interp --with-pydebug + use_clang: true + run_tests: false + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + - name: Build + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} + export PATH="$(llvm-config-${{ env.LLVM_VERSION }} --bindir):$PATH" + if [ "${{ matrix.use_clang }}" = "true" ]; then + export CC=clang-${{ env.LLVM_VERSION }} + fi + ./configure ${{ matrix.configure_flags }} + make all --jobs 4 + - name: Test + if: matrix.run_tests != false + run: | + ${{ matrix.test_env }} ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + continue-on-error: ${{ matrix.continue_on_error }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d74ce8fcc256dc8..e9a4eb2b0808cb7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,10 +19,7 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - uses: pre-commit/action@v3.0.1 + - uses: j178/prek-action@0bb87d7f00b0c99306c8bcb8b8beba1eb581c037 # v1.1.1 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 95133c1338b6823..d748b6ff63e68a1 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -13,14 +13,21 @@ on: - "Lib/test/libregrtest/**" - "Lib/tomllib/**" - "Misc/mypy/**" + - "Tools/build/check_extension_modules.py" + - "Tools/build/check_warnings.py" - "Tools/build/compute-changes.py" + - "Tools/build/consts_getter.py" - "Tools/build/deepfreeze.py" - - "Tools/build/generate_sbom.py" - "Tools/build/generate-build-details.py" - - "Tools/build/verify_ensurepip_wheels.py" - - "Tools/build/update_file.py" + - "Tools/build/generate_levenshtein_examples.py" + - "Tools/build/generate_sbom.py" + - "Tools/build/generate_stdlib_module_names.py" + - "Tools/build/mypy.ini" - "Tools/build/umarshal.py" + - "Tools/build/update_file.py" + - "Tools/build/verify_ensurepip_wheels.py" - "Tools/cases_generator/**" + - "Tools/check-c-api-docs/**" - "Tools/clinic/**" - "Tools/jit/**" - "Tools/peg_generator/**" @@ -53,19 +60,20 @@ jobs: "Lib/tomllib", "Tools/build", "Tools/cases_generator", + "Tools/check-c-api-docs", "Tools/clinic", "Tools/jit", "Tools/peg_generator", ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: - python-version: "3.13" - cache: pip - cache-dependency-path: Tools/requirements-dev.txt - - run: pip install -r Tools/requirements-dev.txt + python-version: "3.15" + activate-environment: true + cache-dependency-glob: Tools/requirements-dev.txt + - run: uv pip install -r Tools/requirements-dev.txt - run: python3 Misc/mypy/make_symlinks.py --symlink - - run: mypy --config-file ${{ matrix.target }}/mypy.ini + - run: mypy --num-workers 4 --config-file ${{ matrix.target }}/mypy.ini diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml index 9f1a8a824e5f19a..1267361040c81b6 100644 --- a/.github/workflows/new-bugs-announce-notifier.yml +++ b/.github/workflows/new-bugs-announce-notifier.yml @@ -6,19 +6,21 @@ on: - opened permissions: - issues: read + contents: read jobs: notify-new-bugs-announce: runs-on: ubuntu-latest + permissions: + issues: read timeout-minutes: 10 steps: - - uses: actions/setup-node@v4 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: 20 - run: npm install mailgun.js form-data - name: Send notification - uses: actions/github-script@v7 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: MAILGUN_API_KEY: ${{ secrets.MAILGUN_PYTHON_ORG_MAILGUN_KEY }} with: @@ -44,7 +46,7 @@ jobs: // We need to truncate the body size, because the max size for // the whole payload is 16kb. We want to be safe and assume that // body can take up to ~8kb of space. - body : issue.data.body.substring(0, 8000) + body : (issue.data.body || "").substring(0, 8000) }; const data = { diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index 0b64367e6c45624..f241fccdeb2a32a 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -1,10 +1,9 @@ #!/bin/sh apt-get update -apt-get -yq install \ +apt-get -yq --no-install-recommends install \ build-essential \ pkg-config \ - ccache \ cmake \ gdb \ lcov \ @@ -27,9 +26,16 @@ apt-get -yq install \ xvfb \ zlib1g-dev -# Workaround missing libmpdec-dev on ubuntu 24.04: -# https://launchpad.net/~ondrej/+archive/ubuntu/php -# https://deb.sury.org/ -sudo add-apt-repository ppa:ondrej/php -apt-get update -apt-get -yq install libmpdec-dev +# Workaround missing libmpdec-dev on ubuntu 24.04 by building mpdecimal +# from source. ppa:ondrej/php (launchpad.net) are unreliable +# (https://status.canonical.com) so fetch the tarball directly +# from the upstream host. +# https://www.bytereef.org/mpdecimal/ +MPDECIMAL_VERSION=4.0.1 +curl -fsSL "https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-${MPDECIMAL_VERSION}.tar.gz" \ + | tar -xz -C /tmp +(cd "/tmp/mpdecimal-${MPDECIMAL_VERSION}" \ + && ./configure --prefix=/usr/local \ + && make -j"$(nproc)" \ + && make install) +ldconfig diff --git a/.github/workflows/project-updater.yml b/.github/workflows/project-updater.yml deleted file mode 100644 index 1d9d637ec848a61..000000000000000 --- a/.github/workflows/project-updater.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Update GH projects - -on: - issues: - types: - - opened - - labeled - -permissions: - contents: read - -jobs: - add-to-project: - name: Add issues to projects - runs-on: ubuntu-latest - timeout-minutes: 10 - strategy: - fail-fast: false - matrix: - include: - # if an issue has any of these labels, it will be added - # to the corresponding project - - { project: 2, label: "release-blocker, deferred-blocker" } - - { project: 32, label: sprint } - - steps: - - uses: actions/add-to-project@v1.0.0 - with: - project-url: https://github.com/orgs/python/projects/${{ matrix.project }} - github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} - labeled: ${{ matrix.label }} diff --git a/.github/workflows/regen-abidump.sh b/.github/workflows/regen-abidump.sh index 251bb3857ecfcb6..75a1a72e3702024 100644 --- a/.github/workflows/regen-abidump.sh +++ b/.github/workflows/regen-abidump.sh @@ -2,7 +2,7 @@ set -ex export DEBIAN_FRONTEND=noninteractive ./.github/workflows/posix-deps-apt.sh -apt-get install -yq abigail-tools python3 +apt-get install -yq --no-install-recommends abigail-tools python3 export CFLAGS="-g3 -O0" ./configure --enable-shared && make make regen-abidump diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index 7e534c58c798d1c..f3e2666879530f9 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -4,6 +4,9 @@ on: pull_request: types: [opened, reopened, labeled, unlabeled, synchronize] +permissions: + contents: read + jobs: label-dnm: name: DO-NOT-MERGE @@ -15,7 +18,7 @@ jobs: steps: - name: Check there's no DO-NOT-MERGE - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 0 @@ -33,7 +36,7 @@ jobs: steps: # Check that the PR is not awaiting changes from the author due to previous review. - name: Check there's no required changes - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 0 @@ -42,7 +45,7 @@ jobs: awaiting change review - id: is-feature name: Check whether this PR is a feature (contains a "type-feature" label) - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 1 @@ -53,7 +56,7 @@ jobs: - id: awaiting-merge if: steps.is-feature.outputs.status == 'success' name: Check for complete review - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 1 diff --git a/.github/workflows/reusable-check-c-api-docs.yml b/.github/workflows/reusable-check-c-api-docs.yml new file mode 100644 index 000000000000000..49e5ef7f768b799 --- /dev/null +++ b/.github/workflows/reusable-check-c-api-docs.yml @@ -0,0 +1,25 @@ +name: Reusable C API Docs Check + +on: + workflow_call: + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + check-c-api-docs: + name: 'Check if all C APIs are documented' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.x' + - name: Check for undocumented C APIs + run: python Tools/check-c-api-docs/main.py diff --git a/.github/workflows/reusable-check-html-ids.yml b/.github/workflows/reusable-check-html-ids.yml new file mode 100644 index 000000000000000..4f827c55cacd067 --- /dev/null +++ b/.github/workflows/reusable-check-html-ids.yml @@ -0,0 +1,67 @@ +name: Reusable check HTML IDs + +on: + workflow_call: + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + check-html-ids: + name: 'Check for removed HTML IDs' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: 'Check out PR head' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha }} + - name: 'Find merge base' + id: merge-base + run: | + BASE="${{ github.event.pull_request.base.sha }}" + HEAD="${{ github.event.pull_request.head.sha }}" + git fetch --depth=$((${{ github.event.pull_request.commits }} + 10)) --no-tags origin "$BASE" "$HEAD" + + if ! MERGE_BASE=$(git merge-base "$BASE" "$HEAD" 2>/dev/null); then + git fetch --deepen=1 --no-tags origin "$BASE" "$HEAD" + + OLDEST=$(git rev-list --reflog --max-parents=0 --reverse "${BASE}^" "${HEAD}^" | head -1) + TIMESTAMP=$(git show --format=%at --no-patch "$OLDEST") + + git fetch --shallow-since="$TIMESTAMP" --no-tags origin "$BASE" "$HEAD" + + MERGE_BASE=$(git merge-base "$BASE" "$HEAD") + fi + echo "sha=$MERGE_BASE" >> "$GITHUB_OUTPUT" + - name: 'Create worktree at merge base' + env: + MERGE_BASE: ${{ steps.merge-base.outputs.sha }} + run: git worktree add /tmp/merge-base "$MERGE_BASE" --detach + - name: 'Set up Python' + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3' + cache: 'pip' + cache-dependency-path: 'Doc/requirements.txt' + - name: 'Install build dependencies' + run: make -C /tmp/merge-base/Doc/ venv + - name: 'Build HTML documentation' + run: make -C /tmp/merge-base/Doc/ SPHINXOPTS="--quiet" html + - name: 'Collect HTML IDs' + run: python Doc/tools/check-html-ids.py collect /tmp/merge-base/Doc/build/html -o /tmp/html-ids-base.json.gz + - name: 'Download PR head HTML IDs' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: html-ids-head.json.gz + path: /tmp + - name: 'Check for removed HTML IDs' + run: | + # shellcheck disable=SC2046 + python Doc/tools/check-html-ids.py -v check \ + /tmp/html-ids-base.json.gz /tmp/html-ids-head.json.gz \ + $([ -f Doc/tools/removed-ids.txt ] && echo "--exclude-file Doc/tools/removed-ids.txt") diff --git a/.github/workflows/reusable-cifuzz.yml b/.github/workflows/reusable-cifuzz.yml new file mode 100644 index 000000000000000..0d02232686339bf --- /dev/null +++ b/.github/workflows/reusable-cifuzz.yml @@ -0,0 +1,49 @@ +# CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ +name: Reusable CIFuzz + +on: + workflow_call: + inputs: + oss-fuzz-project-name: + description: OSS-Fuzz project name + required: true + type: string + sanitizer: + description: OSS-Fuzz sanitizer + required: true + type: string + +permissions: + contents: read + +jobs: + cifuzz: + name: ${{ inputs.oss-fuzz-project-name }} (${{ inputs.sanitizer }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Build fuzzers (${{ inputs.sanitizer }}) + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@ed23f8af80ff82b25ca67cd9b101e690b8897b3f # master + with: + oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }} + sanitizer: ${{ inputs.sanitizer }} + - name: Run fuzzers (${{ inputs.sanitizer }}) + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@ed23f8af80ff82b25ca67cd9b101e690b8897b3f # master + with: + fuzz-seconds: 600 + oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }} + output-sarif: true + sanitizer: ${{ inputs.sanitizer }} + - name: Upload crash + if: failure() && steps.build.outcome == 'success' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: ${{ inputs.sanitizer }}-artifacts + path: ./out/artifacts + - name: Upload SARIF + if: always() && steps.build.outcome == 'success' + uses: github/codeql-action/upload-sarif@38697555549f1db7851b81482ff19f1fa5c4fedc # v4.34.1 + with: + sarif_file: cifuzz-sarif/results.sarif + checkout_path: cifuzz-sarif diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index d2668ddcac1a3d8..b8a9e2960eca591 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -17,24 +17,45 @@ on: # yamllint disable-line rule:truthy # || 'falsy-branch' # }} # - config-hash: - description: Config hash value for use in cache keys - value: ${{ jobs.compute-changes.outputs.config-hash }} # str + run-android: + description: Whether to run the Android tests + value: ${{ jobs.compute-changes.outputs.run-android }} # bool + run-ci-fuzz: + description: Whether to run the CIFuzz job for 'cpython' fuzzer + value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }} # bool + run-ci-fuzz-stdlib: + description: Whether to run the CIFuzz job for 'python3-libraries' fuzzer + value: ${{ jobs.compute-changes.outputs.run-ci-fuzz-stdlib }} # bool run-docs: description: Whether to build the docs value: ${{ jobs.compute-changes.outputs.run-docs }} # bool + run-ios: + description: Whether to run the iOS tests + value: ${{ jobs.compute-changes.outputs.run-ios }} # bool + run-macos: + description: Whether to run the macOS tests + value: ${{ jobs.compute-changes.outputs.run-macos }} # bool run-tests: description: Whether to run the regular tests value: ${{ jobs.compute-changes.outputs.run-tests }} # bool - run-windows-tests: - description: Whether to run the Windows tests - value: ${{ jobs.compute-changes.outputs.run-windows-tests }} # bool + run-ubuntu: + description: Whether to run the Ubuntu tests + value: ${{ jobs.compute-changes.outputs.run-ubuntu }} # bool + run-emscripten: + description: Whether to run the Emscripten tests + value: ${{ jobs.compute-changes.outputs.run-emscripten }} # bool + run-wasi: + description: Whether to run the WASI tests + value: ${{ jobs.compute-changes.outputs.run-wasi }} # bool run-windows-msi: description: Whether to run the MSI installer smoke tests value: ${{ jobs.compute-changes.outputs.run-windows-msi }} # bool - run-ci-fuzz: - description: Whether to run the CIFuzz job - value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }} # bool + run-windows-tests: + description: Whether to run the Windows tests + value: ${{ jobs.compute-changes.outputs.run-windows-tests }} # bool + +permissions: + contents: read jobs: compute-changes: @@ -42,22 +63,28 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 outputs: - config-hash: ${{ steps.config-hash.outputs.hash }} + run-android: ${{ steps.changes.outputs.run-android }} run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }} + run-ci-fuzz-stdlib: ${{ steps.changes.outputs.run-ci-fuzz-stdlib }} run-docs: ${{ steps.changes.outputs.run-docs }} + run-ios: ${{ steps.changes.outputs.run-ios }} + run-macos: ${{ steps.changes.outputs.run-macos }} run-tests: ${{ steps.changes.outputs.run-tests }} + run-ubuntu: ${{ steps.changes.outputs.run-ubuntu }} + run-emscripten: ${{ steps.changes.outputs.run-emscripten }} + run-wasi: ${{ steps.changes.outputs.run-wasi }} run-windows-msi: ${{ steps.changes.outputs.run-windows-msi }} run-windows-tests: ${{ steps.changes.outputs.run-windows-tests }} steps: - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3" - run: >- echo '${{ github.event_name }}' - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false ref: >- @@ -100,8 +127,3 @@ jobs: GITHUB_EVENT_NAME: ${{ github.event_name }} CCF_TARGET_REF: ${{ github.base_ref || github.event.repository.default_branch }} CCF_HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }} - - - name: Compute hash for config cache key - id: config-hash - run: | - echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 7b9dc4818577ebc..7b524569f85c9e8 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -27,7 +27,7 @@ jobs: refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' steps: - name: 'Check out latest PR branch commit' - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false ref: >- @@ -52,11 +52,11 @@ jobs: git fetch origin "${refspec_base}" --shallow-since="${DATE}" \ --no-tags --prune --no-recurse-submodules - name: 'Set up Python' - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3' cache: 'pip' - cache-dependency-path: 'Doc/requirements.txt' + cache-dependency-path: 'Doc/pylock.toml' - name: 'Install build dependencies' run: make -C Doc/ venv @@ -75,18 +75,22 @@ jobs: --fail-if-regression \ --fail-if-improved \ --fail-if-new-news-nit - - name: 'Build EPUB documentation' - continue-on-error: true - run: | - set -Eeuo pipefail - make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet" epub - pip install epubcheck - epubcheck Doc/build/epub/Python.epub &> Doc/epubcheck.txt - - name: 'Check for fatal errors in EPUB' + - name: 'Collect HTML IDs' if: github.event_name == 'pull_request' - continue-on-error: true # until gh-136155 is fixed - run: | - python Doc/tools/check-epub.py + run: python Doc/tools/check-html-ids.py collect Doc/build/html -o Doc/build/html-ids-head.json.gz + - name: 'Upload HTML IDs' + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: html-ids-head + path: Doc/build/html-ids-head.json.gz + archive: false + + check-html-ids: + name: 'Check for removed HTML IDs' + needs: build-doc + if: github.event_name == 'pull_request' + uses: ./.github/workflows/reusable-check-html-ids.yml # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: @@ -94,17 +98,17 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/cache@v4 + - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ~/.cache/pip key: ubuntu-doc-${{ hashFiles('Doc/requirements.txt') }} restore-keys: | ubuntu-doc- - name: 'Install Dependencies' - run: sudo ./.github/workflows/posix-deps-apt.sh && sudo apt-get install wamerican + run: sudo ./.github/workflows/posix-deps-apt.sh && sudo apt-get install --no-install-recommends wamerican - name: 'Configure CPython' run: ./configure --with-pydebug - name: 'Build CPython' @@ -114,3 +118,30 @@ jobs: # Use "xvfb-run" since some doctest tests open GUI windows - name: 'Run documentation doctest' run: xvfb-run make -C Doc/ PYTHON=../python SPHINXERRORHANDLING="--fail-on-warning" doctest + + check-epub: + name: 'Check EPUB' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: 'Set up Python' + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3' + cache: 'pip' + cache-dependency-path: 'Doc/requirements.txt' + - name: 'Install build dependencies' + run: | + make -C Doc/ venv + python -m pip install epubcheck + - name: 'Build EPUB documentation' + run: make -C Doc/ PYTHON=../python epub + - name: 'Run epubcheck' + continue-on-error: true + run: epubcheck Doc/build/epub/Python.epub &> Doc/epubcheck.txt + - run: cat Doc/epubcheck.txt + - name: 'Check for fatal errors in EPUB' + run: python Doc/tools/check-epub.py diff --git a/.github/workflows/reusable-emscripten.yml b/.github/workflows/reusable-emscripten.yml new file mode 100644 index 000000000000000..69a780a9aebc25e --- /dev/null +++ b/.github/workflows/reusable-emscripten.yml @@ -0,0 +1,77 @@ +name: Reusable Emscripten + +on: + workflow_call: + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + build-emscripten-reusable: + name: 'build and test' + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: "Read Emscripten config" + id: emscripten-config + shell: python + run: | + import hashlib + import json + import os + import tomllib + from pathlib import Path + + config = tomllib.loads(Path("Platforms/emscripten/config.toml").read_text()) + h = hashlib.sha256() + h.update(json.dumps(config["dependencies"], sort_keys=True).encode()) + h.update(Path("Platforms/emscripten/make_libffi.sh").read_bytes()) + h.update(b'1') # Update to explicitly bust cache + emsdk_cache = Path(os.environ["RUNNER_TEMP"]) / "emsdk-cache" + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"emscripten-version={config['emscripten-version']}\n") + f.write(f"node-version={config['node-version']}\n") + f.write(f"deps-hash={h.hexdigest()}\n") + with open(os.environ["GITHUB_ENV"], "a") as f: + f.write(f"EMSDK_CACHE={emsdk_cache}\n") + - name: "Install Node.js" + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: ${{ steps.emscripten-config.outputs.node-version }} + - name: "Cache Emscripten SDK" + id: emsdk-cache + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: ${{ env.EMSDK_CACHE }} + key: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }}-${{ steps.emscripten-config.outputs.deps-hash }} + restore-keys: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }} + - name: "Install Python" + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.x' + - name: "Runner image version" + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: "Install Emscripten" + run: python3 Platforms/emscripten install-emscripten + - name: "Configure build Python" + run: python3 Platforms/emscripten configure-build-python -- --config-cache --with-pydebug + - name: "Make build Python" + run: python3 Platforms/emscripten make-build-python + - name: "Make dependencies" + run: >- + python3 Platforms/emscripten make-dependencies + ${{ steps.emsdk-cache.outputs.cache-hit == 'true' && '--check-up-to-date' || '' }} + - name: "Configure host Python" + run: python3 Platforms/emscripten configure-host --host-runner node -- --config-cache + - name: "Make host Python" + run: python3 Platforms/emscripten make-host + - name: "Display build info" + run: python3 Platforms/emscripten run --pythoninfo + - name: "Test" + run: python3 Platforms/emscripten run --test diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index de0c40221364ad9..65a7f857fc4c779 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -3,9 +3,6 @@ name: Reusable macOS on: workflow_call: inputs: - config_hash: - required: true - type: string free-threading: required: false type: boolean @@ -15,6 +12,9 @@ on: required: true type: string +permissions: + contents: read + env: FORCE_COLOR: 1 @@ -31,21 +31,15 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - name: Install Homebrew dependencies run: | - brew install pkg-config openssl@3.0 xz gdbm tcl-tk@8 make - # Because alternate versions are not symlinked into place by default: - brew link --overwrite tcl-tk@8 + brew bundle --file=Misc/Brewfile + brew install make - name: Configure CPython run: | MACOSX_DEPLOYMENT_TARGET=10.15 \ @@ -58,17 +52,17 @@ jobs: --enable-safety \ ${{ inputs.free-threading && '--disable-gil' || '' }} \ --prefix=/opt/python-dev \ - --with-openssl="$(brew --prefix openssl@3.0)" + --with-openssl="$(brew --prefix openssl@3.5)" - name: Build CPython - if : ${{ inputs.free-threading || inputs.os != 'macos-13' }} + if : ${{ inputs.free-threading || inputs.os != 'macos-26-intel' }} run: gmake -j8 - name: Build CPython for compiler warning check - if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + if : ${{ !inputs.free-threading && inputs.os == 'macos-26-intel' }} run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt - name: Display build info run: make pythoninfo - name: Check compiler warnings - if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + if : ${{ !inputs.free-threading && inputs.os == 'macos-26-intel' }} run: >- python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output_macos.txt diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index e6ff02e4838ee66..ef36447964cf418 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -6,15 +6,15 @@ on: sanitizer: required: true type: string - config_hash: - required: true - type: string free-threading: description: Whether to use free-threaded mode required: false type: boolean default: false +permissions: + contents: read + env: FORCE_COLOR: 1 @@ -29,33 +29,26 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.sanitizer }}-${{ inputs.config_hash }} - name: Install dependencies run: | sudo ./.github/workflows/posix-deps-apt.sh # Install clang wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh + sudo ./llvm.sh 20 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-20 100 + sudo update-alternatives --set clang /usr/bin/clang-20 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-20 100 + sudo update-alternatives --set clang++ /usr/bin/clang++-20 if [ "${SANITIZER}" = "TSan" ]; then - sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 - sudo update-alternatives --set clang /usr/bin/clang-17 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 - sudo update-alternatives --set clang++ /usr/bin/clang++-17 # Reduce ASLR to avoid TSan crashing sudo sysctl -w vm.mmap_rnd_bits=28 - else - sudo ./llvm.sh 20 fi - name: Sanitizer option setup @@ -67,21 +60,13 @@ jobs: || '' }}.txt handle_segv=0" >> "$GITHUB_ENV" else - echo "UBSAN_OPTIONS=${SAN_LOG_OPTION}" >> "$GITHUB_ENV" + echo "UBSAN_OPTIONS=${SAN_LOG_OPTION} halt_on_error=1 suppressions=${GITHUB_WORKSPACE}/Tools/ubsan/suppressions.txt" >> "$GITHUB_ENV" fi echo "CC=clang" >> "$GITHUB_ENV" echo "CXX=clang++" >> "$GITHUB_ENV" env: SANITIZER: ${{ inputs.sanitizer }} SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Configure CPython run: >- ./configure @@ -89,7 +74,7 @@ jobs: ${{ inputs.sanitizer == 'TSan' && '--with-thread-sanitizer' - || '--with-undefined-behavior-sanitizer' + || '--with-undefined-behavior-sanitizer --with-strict-overflow' }} --with-pydebug ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} @@ -101,18 +86,18 @@ jobs: run: >- ./python -m test ${{ inputs.sanitizer == 'TSan' && '--tsan' || '' }} - -j4 + -j4 -W --timeout=900 --slowest - name: Parallel tests if: >- inputs.sanitizer == 'TSan' && fromJSON(inputs.free-threading) - run: ./python -m test --tsan-parallel --parallel-threads=4 -j4 + run: ./python -m test --tsan-parallel --parallel-threads=4 -j4 -W --timeout=600 --slowest - name: Display logs if: always() run: find "${GITHUB_WORKSPACE}" -name 'san_log.*' | xargs head -n 1000 - name: Archive logs if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: >- ${{ inputs.sanitizer }}-logs-${{ diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 76b19fd5d1a72e2..f4321cefa1b5985 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -3,9 +3,6 @@ name: Reusable Ubuntu on: workflow_call: inputs: - config_hash: - required: true - type: string bolt-optimizations: description: Whether to enable BOLT optimizations required: false @@ -20,6 +17,14 @@ on: description: OS to run the job required: true type: string + test-opts: + description: Extra options to pass to the test runner via TESTOPTS + required: false + type: string + default: '' + +permissions: + contents: read env: FORCE_COLOR: 1 @@ -30,11 +35,11 @@ jobs: runs-on: ${{ inputs.os }} timeout-minutes: 60 env: - OPENSSL_VER: 3.0.15 + OPENSSL_VER: 3.5.7 PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Register gcc problem matcher @@ -45,7 +50,7 @@ jobs: if: ${{ fromJSON(inputs.bolt-optimizations) }} run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh 19 - sudo apt-get install bolt-19 + sudo apt-get install --no-install-recommends bolt-19 echo PATH="$(llvm-config-19 --bindir):$PATH" >> $GITHUB_ENV - name: Configure OpenSSL env vars run: | @@ -54,21 +59,13 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ inputs.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" @@ -79,11 +76,6 @@ jobs: run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} # `test_unpickle_module_race` writes to the source directory, which is @@ -124,4 +116,6 @@ jobs: run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw - name: Tests working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: xvfb-run make ci + run: xvfb-run make ci EXTRATESTOPTS="${TEST_OPTS}" + env: + TEST_OPTS: ${{ inputs.test-opts }} diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 6beb91e66d40273..48fb70cbff80095 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -2,10 +2,9 @@ name: Reusable WASI on: workflow_call: - inputs: - config_hash: - required: true - type: string + +permissions: + contents: read env: FORCE_COLOR: 1 @@ -13,71 +12,55 @@ env: jobs: build-wasi-reusable: name: 'build and test' - runs-on: ubuntu-24.04 + runs-on: ubuntu-24.04-arm timeout-minutes: 60 env: - WASMTIME_VERSION: 22.0.0 - WASI_SDK_VERSION: 24 - WASI_SDK_PATH: /opt/wasi-sdk + WASMTIME_VERSION: 38.0.3 CROSS_BUILD_PYTHON: cross-build/build CROSS_BUILD_WASI: cross-build/wasm32-wasip1 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false # No problem resolver registered as one doesn't currently exist for Clang. - name: "Install wasmtime" - uses: bytecodealliance/actions/wasmtime/setup@v1 + uses: bytecodealliance/actions/wasmtime/setup@9152e710e9f7182e4c29ad218e4f335a7b203613 # v1.1.3 with: version: ${{ env.WASMTIME_VERSION }} - - name: "Restore WASI SDK" - id: cache-wasi-sdk - uses: actions/cache@v4 - with: - path: ${{ env.WASI_SDK_PATH }} - key: ${{ runner.os }}-wasi-sdk-${{ env.WASI_SDK_VERSION }} - - name: "Install WASI SDK" # Hard-coded to x64. - if: steps.cache-wasi-sdk.outputs.cache-hit != 'true' + - name: "Read WASI SDK version" + id: wasi-sdk-version run: | - mkdir "${WASI_SDK_PATH}" && \ - curl -s -S --location "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz" | \ - tar --strip-components 1 --directory "${WASI_SDK_PATH}" --extract --gunzip - - name: "Configure ccache action" - uses: hendrikmuhs/ccache-action@v1.2 + import tomllib + from pathlib import Path + import os + config = tomllib.loads(Path("Platforms/WASI/config.toml").read_text()) + version = config["targets"]["wasi-sdk"] + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"version={version}\n") + shell: python + - name: "Install WASI SDK" + id: install-wasi-sdk + uses: bytecodealliance/setup-wasi-sdk-action@b2de090b44eb70013ee96b393727d473b35e1728 with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - - name: "Add ccache to PATH" - run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + version: ${{ steps.wasi-sdk-version.outputs.version }} + add-to-path: false - name: "Install Python" - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.x' - name: "Runner image version" run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: "Restore Python build config.cache" - uses: actions/cache@v4 - with: - path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache - # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python. - # Include the hash of `Tools/wasm/wasi.py` as it may change the environment variables. - # (Make sure to keep the key in sync with the other config.cache step below.) - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} - name: "Configure build Python" - run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug + run: python3 Platforms/WASI configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" - run: python3 Tools/wasm/wasi.py make-build-python - - name: "Restore host config.cache" - uses: actions/cache@v4 - with: - path: ${{ env.CROSS_BUILD_WASI }}/config.cache - # Should be kept in sync with the other config.cache step above. - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} + run: python3 Platforms/WASI make-build-python - name: "Configure host" # `--with-pydebug` inferred from configure-build-python - run: python3 Tools/wasm/wasi.py configure-host -- --config-cache + run: python3 Platforms/WASI configure-host -- --config-cache + env: + WASI_SDK_PATH: ${{ steps.install-wasi-sdk.outputs.wasi-sdk-path }} - name: "Make host" - run: python3 Tools/wasm/wasi.py make-host + run: python3 Platforms/WASI make-host - name: "Display build info" run: make --directory "${CROSS_BUILD_WASI}" pythoninfo - name: "Test" diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml index a50de344bba4dac..a74724323ec15f8 100644 --- a/.github/workflows/reusable-windows-msi.yml +++ b/.github/workflows/reusable-windows-msi.yml @@ -17,13 +17,13 @@ env: jobs: build: name: installer for ${{ inputs.arch }} - runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-latest' }} + runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2025-vs2026' }} timeout-minutes: 60 env: ARCH: ${{ inputs.arch }} IncludeFreethreaded: true steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Build CPython installer diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 37c802095b0f2fa..c6e8128884e90c2 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -12,30 +12,38 @@ on: required: false type: boolean default: false + interpreter: + description: Which interpreter to build (switch-case or tail-call) + required: true + type: string + +permissions: + contents: read env: FORCE_COLOR: 1 - IncludeUwp: >- - true jobs: build: - name: Build and test (${{ inputs.arch }}) - runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-latest' }} + name: Build and test (${{ inputs.arch }}, ${{ inputs.interpreter }}) + runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2025-vs2026' }} timeout-minutes: 60 env: ARCH: ${{ inputs.arch }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Register MSVC problem matcher if: inputs.arch != 'Win32' run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython + # msvc::musttail is not supported for debug builds, so we have to + # switch to release. run: >- .\\PCbuild\\build.bat - -e -d -v + -e -v + ${{ inputs.interpreter == 'switch-case' && '-d' || '--tail-call-interp -c Release' }} -p "${ARCH}" ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} shell: bash @@ -45,6 +53,7 @@ jobs: run: >- .\\PCbuild\\rt.bat -p "${ARCH}" - -d -q --fast-ci + -q --fast-ci + ${{ inputs.interpreter == 'switch-case' && '-d' || '' }} ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} shell: bash diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index febb2dd823a8fe2..37f78519dedc329 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -4,23 +4,27 @@ on: schedule: - cron: "0 */6 * * *" +permissions: + contents: read + jobs: stale: if: github.repository_owner == 'python' runs-on: ubuntu-latest permissions: + actions: write pull-requests: write timeout-minutes: 10 steps: - name: "Check PRs" - uses: actions/stale@v9 + uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.' + stale-pr-message: 'This PR is stale because it has been open for 90 days with no activity.' stale-pr-label: 'stale' days-before-issue-stale: -1 - days-before-pr-stale: 30 + days-before-pr-stale: 90 days-before-close: -1 ascending: true operations-per-run: 120 diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index e32cbf0aaa3c3ec..656a14906b3cb79 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -1,19 +1,14 @@ name: Tail calling interpreter on: pull_request: - paths: + paths: &paths - '.github/workflows/tail-call.yml' - 'Python/bytecodes.c' - 'Python/ceval.c' - 'Python/ceval_macros.h' - 'Python/generated_cases.c.h' push: - paths: - - '.github/workflows/tail-call.yml' - - 'Python/bytecodes.c' - - 'Python/ceval.c' - - 'Python/ceval_macros.h' - - 'Python/generated_cases.c.h' + paths: *paths workflow_dispatch: permissions: @@ -25,115 +20,73 @@ concurrency: env: FORCE_COLOR: 1 + LLVM_VERSION: 21 jobs: - tail-call: + macos: name: ${{ matrix.target }} runs-on: ${{ matrix.runner }} - timeout-minutes: 90 + timeout-minutes: 60 strategy: fail-fast: false matrix: - target: -# Un-comment as we add support for more platforms for tail-calling interpreters. -# - i686-pc-windows-msvc/msvc - - x86_64-pc-windows-msvc/msvc -# - aarch64-pc-windows-msvc/msvc - - x86_64-apple-darwin/clang - - aarch64-apple-darwin/clang - - x86_64-unknown-linux-gnu/gcc - - aarch64-unknown-linux-gnu/gcc - - free-threading - llvm: - - 20 include: -# - target: i686-pc-windows-msvc/msvc -# architecture: Win32 -# runner: windows-latest - - target: x86_64-pc-windows-msvc/msvc - architecture: x64 - runner: windows-latest -# - target: aarch64-pc-windows-msvc/msvc -# architecture: ARM64 -# runner: windows-latest - target: x86_64-apple-darwin/clang - architecture: x86_64 - runner: macos-13 + runner: macos-15-intel - target: aarch64-apple-darwin/clang - architecture: aarch64 - runner: macos-14 - - target: x86_64-unknown-linux-gnu/gcc - architecture: x86_64 - runner: ubuntu-24.04 - - target: aarch64-unknown-linux-gnu/gcc - architecture: aarch64 - runner: ubuntu-24.04-arm - - target: free-threading - architecture: x86_64 - runner: ubuntu-24.04 + runner: macos-15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' - - - name: Native Windows (debug) - if: runner.os == 'Windows' && matrix.architecture != 'ARM64' - shell: cmd - run: | - choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 - set PlatformToolset=clangcl - set LLVMToolsVersion=${{ matrix.llvm }}.1.0 - set LLVMInstallDir=C:\Program Files\LLVM - call ./PCbuild/build.bat --tail-call-interp -d -p ${{ matrix.architecture }} - call ./PCbuild/rt.bat -d -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - # No tests (yet): - - name: Emulated Windows (release) - if: runner.os == 'Windows' && matrix.architecture == 'ARM64' - shell: cmd - run: | - choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 - set PlatformToolset=clangcl - set LLVMToolsVersion=${{ matrix.llvm }}.1.0 - set LLVMInstallDir=C:\Program Files\LLVM - ./PCbuild/build.bat --tail-call-interp -p ${{ matrix.architecture }} - - # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. - # This is a bug in the macOS runner image where the pre-installed Python is installed in the same - # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes - # the symlink to the pre-installed Python so that the Homebrew Python is used instead. - # Note: when a new LLVM is released, the homebrew installation directory changes, so the builds will fail. - # We either need to upgrade LLVM or change the directory being pointed to. - - name: Native macOS (release) - if: runner.os == 'macOS' + - name: Install dependencies run: | brew update - find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete - brew install llvm@${{ matrix.llvm }} + brew install llvm@${{ env.LLVM_VERSION }} + - name: Build + run: | export SDKROOT="$(xcrun --show-sdk-path)" - export PATH="/usr/local/opt/llvm/bin:$PATH" - export PATH="/opt/homebrew/opt/llvm/bin:$PATH" - CC=clang-20 ./configure --with-tail-call-interp + export PATH="/usr/local/opt/llvm@${{ env.LLVM_VERSION }}/bin:$PATH" + export PATH="/opt/homebrew/opt/llvm@${{ env.LLVM_VERSION }}/bin:$PATH" + CC=clang-${{ env.LLVM_VERSION }} ./configure --with-tail-call-interp make all --jobs 4 + - name: Test + run: | ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: Native Linux (debug) - if: runner.os == 'Linux' && matrix.target != 'free-threading' + linux: + name: ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-gnu/gcc + runner: ubuntu-24.04 + configure_flags: --with-pydebug + - target: x86_64-unknown-linux-gnu/gcc-free-threading + runner: ubuntu-24.04 + configure_flags: --disable-gil + - target: aarch64-unknown-linux-gnu/gcc + runner: ubuntu-24.04-arm + configure_flags: --with-pydebug + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - name: Build run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - CC=clang-20 ./configure --with-tail-call-interp --with-pydebug + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} + export PATH="$(llvm-config-${{ env.LLVM_VERSION }} --bindir):$PATH" + CC=clang-${{ env.LLVM_VERSION }} ./configure --with-tail-call-interp ${{ matrix.configure_flags }} make all --jobs 4 - ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - - name: Native Linux with free-threading (release) - if: matrix.target == 'free-threading' + - name: Test run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - CC=clang-20 ./configure --with-tail-call-interp --disable-gil - make all --jobs 4 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 463e7bf3355cc39..cb40f6abc0b3b75 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -25,10 +25,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3' - name: Compare checksum of bundled wheels to the ones published on PyPI diff --git a/.github/workflows/verify-expat.yml b/.github/workflows/verify-expat.yml new file mode 100644 index 000000000000000..472a11db2da5fbf --- /dev/null +++ b/.github/workflows/verify-expat.yml @@ -0,0 +1,32 @@ +name: Verify bundled libexpat + +on: + workflow_dispatch: + push: + paths: + - 'Modules/expat/**' + - '.github/workflows/verify-expat.yml' + pull_request: + paths: + - 'Modules/expat/**' + - '.github/workflows/verify-expat.yml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Download and verify bundled libexpat files + run: | + ./Modules/expat/refresh.sh + git diff --exit-code Modules/expat/ diff --git a/.github/zizmor.yml b/.github/zizmor.yml index 9b42b47cc855456..7c776d5ea1f941a 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -1,10 +1,6 @@ -# Configuration for the zizmor static analysis tool, run via pre-commit in CI -# https://woodruffw.github.io/zizmor/configuration/ +# Configuration for the zizmor static analysis tool, run via prek in CI +# https://docs.zizmor.sh/configuration/ rules: dangerous-triggers: ignore: - documentation-links.yml - unpinned-uses: - config: - policies: - "*": ref-pin diff --git a/.gitignore b/.gitignore index 7aa6272cf8e3828..118eb5ee76e8051 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ *.cover *.iml *.o +*.o.tmp *.lto *.a *.so @@ -45,6 +46,7 @@ gmon.out .pytest_cache/ .ruff_cache/ .DS_Store +.pixi/ *.exe @@ -71,16 +73,15 @@ Lib/test/data/* /Makefile /Makefile.pre /iOSTestbed.* -iOS/Frameworks/ -iOS/Resources/Info.plist -iOS/testbed/build -iOS/testbed/Python.xcframework/ios-*/bin -iOS/testbed/Python.xcframework/ios-*/include -iOS/testbed/Python.xcframework/ios-*/lib -iOS/testbed/Python.xcframework/ios-*/Python.framework -iOS/testbed/iOSTestbed.xcodeproj/project.xcworkspace -iOS/testbed/iOSTestbed.xcodeproj/xcuserdata -iOS/testbed/iOSTestbed.xcodeproj/xcshareddata +Apple/iOS/Frameworks/ +Apple/iOS/Resources/Info.plist +Apple/testbed/build +Apple/testbed/Python.xcframework/*/bin +Apple/testbed/Python.xcframework/*/include +Apple/testbed/Python.xcframework/*/lib +Apple/testbed/Python.xcframework/*/Python.framework +Apple/testbed/*Testbed.xcodeproj/project.xcworkspace +Apple/testbed/*Testbed.xcodeproj/xcuserdata Mac/Makefile Mac/PythonLauncher/Info.plist Mac/PythonLauncher/Makefile @@ -136,10 +137,11 @@ Tools/unicode/data/ /config.log /config.status /config.status.lineno -# hendrikmuhs/ccache-action@v1 /.ccache -/cross-build/ +/cross-build*/ /jit_stencils*.h +/jit_unwind_info*.h +.jit-stamp /platform /profile-clean-stamp /profile-run-stamp diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 86410c46d1d707b..6c7f1b93e66e879 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,48 +1,89 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.8 + rev: 3b3f7c3f57fe9925356faf5fe6230835138be230 # frozen: v0.15.17 hooks: - - id: ruff + - id: ruff-check + name: Run Ruff (lint) on Platforms/Apple/ + args: [--exit-non-zero-on-fix, --config=Platforms/Apple/.ruff.toml] + files: ^Platforms/Apple/ + - id: ruff-check name: Run Ruff (lint) on Doc/ args: [--exit-non-zero-on-fix] files: ^Doc/ - - id: ruff + - id: ruff-check + name: Run Ruff (lint) on Lib/ + args: [--exit-non-zero-on-fix] + files: ^Lib/ + exclude: ^Lib/test/ + - id: ruff-check name: Run Ruff (lint) on Lib/test/ args: [--exit-non-zero-on-fix] files: ^Lib/test/ - - id: ruff + - id: ruff-check + name: Run Ruff (lint) on Platforms/WASI/ + args: [--exit-non-zero-on-fix, --config=Platforms/WASI/.ruff.toml] + files: ^Platforms/WASI/ + - id: ruff-check + name: Run Ruff (lint) on Tools/ + args: [--exit-non-zero-on-fix] + files: ^Tools/ + exclude: ^Tools/(build|clinic|i18n|peg_generator|wasm)/ + - id: ruff-check name: Run Ruff (lint) on Tools/build/ args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] files: ^Tools/build/ - - id: ruff + - id: ruff-check + name: Run Ruff (lint) on Tools/i18n/ + args: [--exit-non-zero-on-fix, --config=Tools/i18n/.ruff.toml] + files: ^Tools/i18n/ + - id: ruff-check name: Run Ruff (lint) on Argument Clinic args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml] files: ^Tools/clinic/|Lib/test/test_clinic.py + - id: ruff-check + name: Run Ruff (lint) on Tools/peg_generator/ + args: [--exit-non-zero-on-fix, --config=Tools/peg_generator/.ruff.toml] + files: ^Tools/peg_generator/ + - id: ruff-check + name: Run Ruff (lint) on Tools/wasm/ + args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml] + files: ^Tools/wasm/ + - id: ruff-format + name: Run Ruff (format) on Platforms/Apple/ + args: [--exit-non-zero-on-fix, --config=Platforms/Apple/.ruff.toml] + files: ^Platforms/Apple/ - id: ruff-format name: Run Ruff (format) on Doc/ - args: [--check] + args: [--exit-non-zero-on-fix] files: ^Doc/ + - id: ruff-format + name: Run Ruff (format) on Platforms/WASI/ + args: [--exit-non-zero-on-fix, --config=Platforms/WASI/.ruff.toml] + files: ^Platforms/WASI/ - id: ruff-format name: Run Ruff (format) on Tools/build/check_warnings.py - args: [--check, --config=Tools/build/.ruff.toml] + args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] files: ^Tools/build/check_warnings.py + - id: ruff-format + name: Run Ruff (format) on Tools/wasm/ + args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml] + files: ^Tools/wasm/ - repo: https://github.com/psf/black-pre-commit-mirror - rev: 25.1.0 + rev: ea488cebbfd88a5f50b8bd95d5c829d0bb76feb8 # frozen: 26.1.0 hooks: - id: black name: Run Black on Tools/jit/ files: ^Tools/jit/ - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.5.5 + rev: ad1b27d73581aa16cca06fc4a0761fc563ffe8e8 # frozen: v1.5.6 hooks: - id: remove-tabs types: [python] - exclude: ^Tools/c-analyzer/cpython/_parser.py - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0 hooks: - id: check-case-conflict - id: check-merge-conflict @@ -54,30 +95,33 @@ repos: exclude: Lib/test/tokenizedata/coding20731.py - id: end-of-file-fixer files: '^\.github/CODEOWNERS$' + - id: mixed-line-ending + args: [--fix=auto] + exclude: '^Lib/test/.*data/' - id: trailing-whitespace types_or: [c, inc, python, rst, yaml] - id: trailing-whitespace files: '^\.github/CODEOWNERS|\.(gram)$' - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.33.0 + rev: 9f48a48aa91a6040d749ad68ec70907d907a5a7f # frozen: 0.37.0 hooks: - id: check-dependabot - id: check-github-workflows - id: check-readthedocs - repo: https://github.com/rhysd/actionlint - rev: v1.7.7 + rev: 393031adb9afb225ee52ae2ccd7a5af5525e03e8 # frozen: v1.7.11 hooks: - id: actionlint - - repo: https://github.com/woodruffw/zizmor-pre-commit - rev: v1.6.0 + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: b546b77c44c466a54a42af5499dcc0dcc1a3193f # frozen: v1.22.0 hooks: - id: zizmor - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v1.0.0 + rev: c883505f64b59c3c5c9375191e4ad9f98e727ccd # frozen: v1.0.2 hooks: - id: sphinx-lint args: [--enable=default-role] diff --git a/.readthedocs.yml b/.readthedocs.yml index 0a2c3f8345367f4..038417e4bb34385 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -11,24 +11,50 @@ build: os: ubuntu-24.04 tools: python: "3" + apt_packages: + - jq - commands: - # https://docs.readthedocs.io/en/stable/build-customization.html#cancel-build-based-on-a-condition - # - # Cancel building pull requests when there aren't changes in the Doc directory. - # - # If there are no changes (git diff exits with 0) we force the command to return with 183. - # This is a special exit code on Read the Docs that will cancel the build immediately. - - | - if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && [ "$(git diff --quiet origin/main -- Doc/ .readthedocs.yml; echo $?)" -eq 0 ]; - then - echo "No changes to Doc/ - exiting the build."; - exit 183; - fi - - - asdf plugin add uv - - asdf install uv latest - - asdf global uv latest - - make -C Doc venv html - - mkdir _readthedocs - - mv Doc/build/html _readthedocs/html + jobs: + post_system_dependencies: + # https://docs.readthedocs.com/platform/stable/guides/build/skip-build.html#skip-builds-based-on-conditions + # + # Cancel building pull requests when there are no changes in the Doc + # directory or RTD configuration, or if we can't cleanly merge the base + # branch. + - | + set -eEux; + if [ "$READTHEDOCS_VERSION_TYPE" = "external" ]; + then + base_branch=$(wget -qO- "https://api.github.com/repos/python/cpython/pulls/$READTHEDOCS_VERSION" | jq -er ".base.ref"); + git fetch --depth=50 origin $base_branch:origin-$base_branch; + for attempt in $(seq 10); + do + if ! git merge-base HEAD origin-$base_branch; + then + git fetch --deepen=50 origin $base_branch; + else + break; + fi; + done; + if ! git -c "user.name=rtd" -c "user.email=no-reply@readthedocs.org" merge --no-stat --no-edit origin-$base_branch; + then + echo "Unsuccessful merge with '$base_branch' branch, skipping the build"; + exit 183; + fi; + if git diff --exit-code --stat origin-$base_branch -- Doc/ .readthedocs.yml; + then + echo "No changes to Doc/ - skipping the build."; + exit 183; + fi; + fi; + create_environment: + - echo "Skipping default environment creation" + install: + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest + build: + html: + - make -C Doc venv html + - mkdir -p "$READTHEDOCS_OUTPUT" + - mv Doc/build/html "$READTHEDOCS_OUTPUT/" diff --git a/.well-known/funding-manifest-urls b/.well-known/funding-manifest-urls new file mode 100644 index 000000000000000..d56ca9eb52f0040 --- /dev/null +++ b/.well-known/funding-manifest-urls @@ -0,0 +1 @@ +https://www.python.org/funding.json diff --git a/Android/README.md b/Android/README.md deleted file mode 100644 index c42eb627006e6aa..000000000000000 --- a/Android/README.md +++ /dev/null @@ -1,166 +0,0 @@ -# Python for Android - -If you obtained this README as part of a release package, then the only -applicable sections are "Prerequisites", "Testing", and "Using in your own app". - -If you obtained this README as part of the CPython source tree, then you can -also follow the other sections to compile Python for Android yourself. - -However, most app developers should not need to do any of these things manually. -Instead, use one of the tools listed -[here](https://docs.python.org/3/using/android.html), which will provide a much -easier experience. - - -## Prerequisites - -If you already have an Android SDK installed, export the `ANDROID_HOME` -environment variable to point at its location. Otherwise, here's how to install -it: - -* Download the "Command line tools" from . -* Create a directory `android-sdk/cmdline-tools`, and unzip the command line - tools package into it. -* Rename `android-sdk/cmdline-tools/cmdline-tools` to - `android-sdk/cmdline-tools/latest`. -* `export ANDROID_HOME=/path/to/android-sdk` - -The `android.py` script will automatically use the SDK's `sdkmanager` to install -any packages it needs. - -The script also requires the following commands to be on the `PATH`: - -* `curl` -* `java` (or set the `JAVA_HOME` environment variable) - - -## Building - -Python can be built for Android on any POSIX platform supported by the Android -development tools, which currently means Linux or macOS. - -First we'll make a "build" Python (for your development machine), then use it to -help produce a "host" Python for Android. So make sure you have all the usual -tools and libraries needed to build Python for your development machine. - -The easiest way to do a build is to use the `android.py` script. You can either -have it perform the entire build process from start to finish in one step, or -you can do it in discrete steps that mirror running `configure` and `make` for -each of the two builds of Python you end up producing. - -The discrete steps for building via `android.py` are: - -```sh -./android.py configure-build -./android.py make-build -./android.py configure-host HOST -./android.py make-host HOST -``` - -`HOST` identifies which architecture to build. To see the possible values, run -`./android.py configure-host --help`. - -To do all steps in a single command, run: - -```sh -./android.py build HOST -``` - -In the end you should have a build Python in `cross-build/build`, and a host -Python in `cross-build/HOST`. - -You can use `--` as a separator for any of the `configure`-related commands – -including `build` itself – to pass arguments to the underlying `configure` -call. For example, if you want a pydebug build that also caches the results from -`configure`, you can do: - -```sh -./android.py build HOST -- -C --with-pydebug -``` - - -## Packaging - -After building an architecture as described in the section above, you can -package it for release with this command: - -```sh -./android.py package HOST -``` - -`HOST` is defined in the section above. - -This will generate a tarball in `cross-build/HOST/dist`, whose structure is -similar to the `Android` directory of the CPython source tree. - - -## Testing - -The Python test suite can be run on Linux, macOS, or Windows: - -* On Linux, the emulator needs access to the KVM virtualization interface, and - a DISPLAY environment variable pointing at an X server. Xvfb is acceptable. - -The test suite can usually be run on a device with 2 GB of RAM, but this is -borderline, so you may need to increase it to 4 GB. As of Android -Studio Koala, 2 GB is the default for all emulators, although the user interface -may indicate otherwise. Locate the emulator's directory under `~/.android/avd`, -and find `hw.ramSize` in both config.ini and hardware-qemu.ini. Either set these -manually to the same value, or use the Android Studio Device Manager, which will -update both files. - -You can run the test suite either: - -* Within the CPython repository, after doing a build as described above. On - Windows, you won't be able to do the build on the same machine, so you'll have - to copy the `cross-build/HOST/prefix` directory from somewhere else. - -* Or by taking a release package built using the `package` command, extracting - it wherever you want, and using its own copy of `android.py`. - -The test script supports the following modes: - -* In `--connected` mode, it runs on a device or emulator you have already - connected to the build machine. List the available devices with - `$ANDROID_HOME/platform-tools/adb devices -l`, then pass a device ID to the - script like this: - - ```sh - ./android.py test --connected emulator-5554 - ``` - -* In `--managed` mode, it uses a temporary headless emulator defined in the - `managedDevices` section of testbed/app/build.gradle.kts. This mode is slower, - but more reproducible. - - We currently define two devices: `minVersion` and `maxVersion`, corresponding - to our minimum and maximum supported Android versions. For example: - - ```sh - ./android.py test --managed maxVersion - ``` - -By default, the only messages the script will show are Python's own stdout and -stderr. Add the `-v` option to also show Gradle output, and non-Python logcat -messages. - -Any other arguments on the `android.py test` command line will be passed through -to `python -m test` – use `--` to separate them from android.py's own options. -See the [Python Developer's -Guide](https://devguide.python.org/testing/run-write-tests/) for common options -– most of them will work on Android, except for those that involve subprocesses, -such as `-j`. - -Every time you run `android.py test`, changes in pure-Python files in the -repository's `Lib` directory will be picked up immediately. Changes in C files, -and architecture-specific files such as sysconfigdata, will not take effect -until you re-run `android.py make-host` or `build`. - -The testbed app can also be used to test third-party packages. For more details, -run `android.py test --help`, paying attention to the options `--site-packages`, -`--cwd`, `-c` and `-m`. - - -## Using in your own app - -See https://docs.python.org/3/using/android.html. diff --git a/Android/android.py b/Android/android.py deleted file mode 100755 index 75f73cd30993dae..000000000000000 --- a/Android/android.py +++ /dev/null @@ -1,819 +0,0 @@ -#!/usr/bin/env python3 - -import asyncio -import argparse -import os -import re -import shlex -import shutil -import signal -import subprocess -import sys -import sysconfig -from asyncio import wait_for -from contextlib import asynccontextmanager -from datetime import datetime, timezone -from glob import glob -from os.path import abspath, basename, relpath -from pathlib import Path -from subprocess import CalledProcessError -from tempfile import TemporaryDirectory - - -SCRIPT_NAME = Path(__file__).name -ANDROID_DIR = Path(__file__).resolve().parent -PYTHON_DIR = ANDROID_DIR.parent -in_source_tree = ( - ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists() -) - -TESTBED_DIR = ANDROID_DIR / "testbed" -CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" - -HOSTS = ["aarch64-linux-android", "x86_64-linux-android"] -APP_ID = "org.python.testbed" -DECODE_ARGS = ("UTF-8", "backslashreplace") - - -try: - android_home = Path(os.environ['ANDROID_HOME']) -except KeyError: - sys.exit("The ANDROID_HOME environment variable is required.") - -adb = Path( - f"{android_home}/platform-tools/adb" - + (".exe" if os.name == "nt" else "") -) - -gradlew = Path( - f"{TESTBED_DIR}/gradlew" - + (".bat" if os.name == "nt" else "") -) - -# Whether we've seen any output from Python yet. -python_started = False - -# Buffer for verbose output which will be displayed only if a test fails and -# there has been no output from Python. -hidden_output = [] - - -def log_verbose(context, line, stream=sys.stdout): - if context.verbose: - stream.write(line) - else: - hidden_output.append((stream, line)) - - -def delete_glob(pattern): - # Path.glob doesn't accept non-relative patterns. - for path in glob(str(pattern)): - path = Path(path) - print(f"Deleting {path} ...") - if path.is_dir() and not path.is_symlink(): - shutil.rmtree(path) - else: - path.unlink() - - -def subdir(*parts, create=False): - path = CROSS_BUILD_DIR.joinpath(*parts) - if not path.exists(): - if not create: - sys.exit( - f"{path} does not exist. Create it by running the appropriate " - f"`configure` subcommand of {SCRIPT_NAME}.") - else: - path.mkdir(parents=True) - return path - - -def run(command, *, host=None, env=None, log=True, **kwargs): - kwargs.setdefault("check", True) - if env is None: - env = os.environ.copy() - - if host: - host_env = android_env(host) - print_env(host_env) - env.update(host_env) - - if log: - print(">", join_command(command)) - return subprocess.run(command, env=env, **kwargs) - - -# Format a command so it can be copied into a shell. Like shlex.join, but also -# accepts arguments which are Paths, or a single string/Path outside of a list. -def join_command(args): - if isinstance(args, (str, Path)): - return str(args) - else: - return shlex.join(map(str, args)) - - -# Format the environment so it can be pasted into a shell. -def print_env(env): - for key, value in sorted(env.items()): - print(f"export {key}={shlex.quote(value)}") - - -def android_env(host): - if host: - prefix = subdir(host) / "prefix" - else: - prefix = ANDROID_DIR / "prefix" - sysconfig_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py") - sysconfig_filename = next(sysconfig_files).name - host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1] - - env_script = ANDROID_DIR / "android-env.sh" - env_output = subprocess.run( - f"set -eu; " - f"HOST={host}; " - f"PREFIX={prefix}; " - f". {env_script}; " - f"export", - check=True, shell=True, capture_output=True, encoding='utf-8', - ).stdout - - env = {} - for line in env_output.splitlines(): - # We don't require every line to match, as there may be some other - # output from installing the NDK. - if match := re.search( - "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line - ): - key, value = match[2], match[3] - if os.environ.get(key) != value: - env[key] = value - - if not env: - raise ValueError(f"Found no variables in {env_script.name} output:\n" - + env_output) - return env - - -def build_python_path(): - """The path to the build Python binary.""" - build_dir = subdir("build") - binary = build_dir / "python" - if not binary.is_file(): - binary = binary.with_suffix(".exe") - if not binary.is_file(): - raise FileNotFoundError("Unable to find `python(.exe)` in " - f"{build_dir}") - - return binary - - -def configure_build_python(context): - if context.clean: - clean("build") - os.chdir(subdir("build", create=True)) - - command = [relpath(PYTHON_DIR / "configure")] - if context.args: - command.extend(context.args) - run(command) - - -def make_build_python(context): - os.chdir(subdir("build")) - run(["make", "-j", str(os.cpu_count())]) - - -def unpack_deps(host, prefix_dir): - os.chdir(prefix_dir) - deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" - for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.15-4", - "sqlite-3.49.1-0", "xz-5.4.6-1", "zstd-1.5.7-1"]: - filename = f"{name_ver}-{host}.tar.gz" - download(f"{deps_url}/{name_ver}/{filename}") - shutil.unpack_archive(filename) - os.remove(filename) - - -def download(url, target_dir="."): - out_path = f"{target_dir}/{basename(url)}" - run(["curl", "-Lf", "--retry", "5", "--retry-all-errors", "-o", out_path, url]) - return out_path - - -def configure_host_python(context): - if context.clean: - clean(context.host) - - host_dir = subdir(context.host, create=True) - prefix_dir = host_dir / "prefix" - if not prefix_dir.exists(): - prefix_dir.mkdir() - unpack_deps(context.host, prefix_dir) - - os.chdir(host_dir) - command = [ - # Basic cross-compiling configuration - relpath(PYTHON_DIR / "configure"), - f"--host={context.host}", - f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", - f"--with-build-python={build_python_path()}", - "--without-ensurepip", - - # Android always uses a shared libpython. - "--enable-shared", - "--without-static-libpython", - - # Dependent libraries. The others are found using pkg-config: see - # android-env.sh. - f"--with-openssl={prefix_dir}", - ] - - if context.args: - command.extend(context.args) - run(command, host=context.host) - - -def make_host_python(context): - # The CFLAGS and LDFLAGS set in android-env include the prefix dir, so - # delete any previous Python installation to prevent it being used during - # the build. - host_dir = subdir(context.host) - prefix_dir = host_dir / "prefix" - for pattern in ("include/python*", "lib/libpython*", "lib/python*"): - delete_glob(f"{prefix_dir}/{pattern}") - - # The Android environment variables were already captured in the Makefile by - # `configure`, and passing them again when running `make` may cause some - # flags to be duplicated. So we don't use the `host` argument here. - os.chdir(host_dir) - run(["make", "-j", str(os.cpu_count())]) - run(["make", "install", f"prefix={prefix_dir}"]) - - -def build_all(context): - steps = [configure_build_python, make_build_python, configure_host_python, - make_host_python] - for step in steps: - step(context) - - -def clean(host): - delete_glob(CROSS_BUILD_DIR / host) - - -def clean_all(context): - for host in HOSTS + ["build"]: - clean(host) - - -def setup_sdk(): - sdkmanager = android_home / ( - "cmdline-tools/latest/bin/sdkmanager" - + (".bat" if os.name == "nt" else "") - ) - - # Gradle will fail if it needs to install an SDK package whose license - # hasn't been accepted, so pre-accept all licenses. - if not all((android_home / "licenses" / path).exists() for path in [ - "android-sdk-arm-dbt-license", "android-sdk-license" - ]): - run( - [sdkmanager, "--licenses"], - text=True, - capture_output=True, - input="y\n" * 100, - ) - - # Gradle may install this automatically, but we can't rely on that because - # we need to run adb within the logcat task. - if not adb.exists(): - run([sdkmanager, "platform-tools"]) - - -# To avoid distributing compiled artifacts without corresponding source code, -# the Gradle wrapper is not included in the CPython repository. Instead, we -# extract it from the Gradle GitHub repository. -def setup_testbed(): - paths = ["gradlew", "gradlew.bat", "gradle/wrapper/gradle-wrapper.jar"] - if all((TESTBED_DIR / path).exists() for path in paths): - return - - # The wrapper version isn't important, as any version of the wrapper can - # download any version of Gradle. The Gradle version actually used for the - # build is specified in testbed/gradle/wrapper/gradle-wrapper.properties. - version = "8.9.0" - - for path in paths: - out_path = TESTBED_DIR / path - out_path.parent.mkdir(exist_ok=True) - download( - f"https://raw.githubusercontent.com/gradle/gradle/v{version}/{path}", - out_path.parent, - ) - os.chmod(out_path, 0o755) - - -# run_testbed will build the app automatically, but it's useful to have this as -# a separate command to allow running the app outside of this script. -def build_testbed(context): - setup_sdk() - setup_testbed() - run( - [gradlew, "--console", "plain", "packageDebug", "packageDebugAndroidTest"], - cwd=TESTBED_DIR, - ) - - -# Work around a bug involving sys.exit and TaskGroups -# (https://github.com/python/cpython/issues/101515). -def exit(*args): - raise MySystemExit(*args) - - -class MySystemExit(Exception): - pass - - -# The `test` subcommand runs all subprocesses through this context manager so -# that no matter what happens, they can always be cancelled from another task, -# and they will always be cleaned up on exit. -@asynccontextmanager -async def async_process(*args, **kwargs): - process = await asyncio.create_subprocess_exec(*args, **kwargs) - try: - yield process - finally: - if process.returncode is None: - # Allow a reasonably long time for Gradle to clean itself up, - # because we don't want stale emulators left behind. - timeout = 10 - process.terminate() - try: - await wait_for(process.wait(), timeout) - except TimeoutError: - print( - f"Command {args} did not terminate after {timeout} seconds " - f" - sending SIGKILL" - ) - process.kill() - - # Even after killing the process we must still wait for it, - # otherwise we'll get the warning "Exception ignored in __del__". - await wait_for(process.wait(), timeout=1) - - -async def async_check_output(*args, **kwargs): - async with async_process( - *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs - ) as process: - stdout, stderr = await process.communicate() - if process.returncode == 0: - return stdout.decode(*DECODE_ARGS) - else: - raise CalledProcessError( - process.returncode, args, - stdout.decode(*DECODE_ARGS), stderr.decode(*DECODE_ARGS) - ) - - -# Return a list of the serial numbers of connected devices. Emulators will have -# serials of the form "emulator-5678". -async def list_devices(): - serials = [] - header_found = False - - lines = (await async_check_output(adb, "devices")).splitlines() - for line in lines: - # Ignore blank lines, and all lines before the header. - line = line.strip() - if line == "List of devices attached": - header_found = True - elif header_found and line: - try: - serial, status = line.split() - except ValueError: - raise ValueError(f"failed to parse {line!r}") - if status == "device": - serials.append(serial) - - if not header_found: - raise ValueError(f"failed to parse {lines}") - return serials - - -async def find_device(context, initial_devices): - if context.managed: - print("Waiting for managed device - this may take several minutes") - while True: - new_devices = set(await list_devices()).difference(initial_devices) - if len(new_devices) == 0: - await asyncio.sleep(1) - elif len(new_devices) == 1: - serial = new_devices.pop() - print(f"Serial: {serial}") - return serial - else: - exit(f"Found more than one new device: {new_devices}") - else: - return context.connected - - -# An older version of this script in #121595 filtered the logs by UID instead. -# But logcat can't filter by UID until API level 31. If we ever switch back to -# filtering by UID, we'll also have to filter by time so we only show messages -# produced after the initial call to `stop_app`. -# -# We're more likely to miss the PID because it's shorter-lived, so there's a -# workaround in PythonSuite.kt to stop it being *too* short-lived. -async def find_pid(serial): - print("Waiting for app to start - this may take several minutes") - shown_error = False - while True: - try: - # `pidof` requires API level 24 or higher. The level 23 emulator - # includes it, but it doesn't work (it returns all processes). - pid = (await async_check_output( - adb, "-s", serial, "shell", "pidof", "-s", APP_ID - )).strip() - except CalledProcessError as e: - # If the app isn't running yet, pidof gives no output. So if there - # is output, there must have been some other error. However, this - # sometimes happens transiently, especially when running a managed - # emulator for the first time, so don't make it fatal. - if (e.stdout or e.stderr) and not shown_error: - print_called_process_error(e) - print("This may be transient, so continuing to wait") - shown_error = True - else: - # Some older devices (e.g. Nexus 4) return zero even when no process - # was found, so check whether we actually got any output. - if pid: - print(f"PID: {pid}") - return pid - - # Loop fairly rapidly to avoid missing a short-lived process. - await asyncio.sleep(0.2) - - -async def logcat_task(context, initial_devices): - # Gradle may need to do some large downloads of libraries and emulator - # images. This will happen during find_device in --managed mode, or find_pid - # in --connected mode. - startup_timeout = 600 - serial = await wait_for(find_device(context, initial_devices), startup_timeout) - pid = await wait_for(find_pid(serial), startup_timeout) - - # `--pid` requires API level 24 or higher. - args = [adb, "-s", serial, "logcat", "--pid", pid, "--format", "tag"] - logcat_started = False - async with async_process( - *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - ) as process: - while line := (await process.stdout.readline()).decode(*DECODE_ARGS): - if match := re.fullmatch(r"([A-Z])/(.*)", line, re.DOTALL): - logcat_started = True - level, message = match.groups() - else: - # If the regex doesn't match, this is either a logcat startup - # error, or the second or subsequent line of a multi-line - # message. Python won't produce multi-line messages, but other - # components might. - level, message = None, line - - # Exclude high-volume messages which are rarely useful. - if context.verbose < 2 and "from python test_syslog" in message: - continue - - # Put high-level messages on stderr so they're highlighted in the - # buildbot logs. This will include Python's own stderr. - stream = ( - sys.stderr - if level in ["W", "E", "F"] # WARNING, ERROR, FATAL (aka ASSERT) - else sys.stdout - ) - - # To simplify automated processing of the output, e.g. a buildbot - # posting a failure notice on a GitHub PR, we strip the level and - # tag indicators from Python's stdout and stderr. - for prefix in ["python.stdout: ", "python.stderr: "]: - if message.startswith(prefix): - global python_started - python_started = True - stream.write(message.removeprefix(prefix)) - break - else: - # Non-Python messages add a lot of noise, but they may - # sometimes help explain a failure. - log_verbose(context, line, stream) - - # If the device disconnects while logcat is running, which always - # happens in --managed mode, some versions of adb return non-zero. - # Distinguish this from a logcat startup error by checking whether we've - # received any logcat messages yet. - status = await wait_for(process.wait(), timeout=1) - if status != 0 and not logcat_started: - raise CalledProcessError(status, args) - - -def stop_app(serial): - run([adb, "-s", serial, "shell", "am", "force-stop", APP_ID], log=False) - - -async def gradle_task(context): - env = os.environ.copy() - if context.managed: - task_prefix = context.managed - else: - task_prefix = "connected" - env["ANDROID_SERIAL"] = context.connected - - if context.command: - mode = "-c" - module = context.command - else: - mode = "-m" - module = context.module or "test" - - args = [ - gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest", - ] + [ - # Build-time properties - f"-Ppython.{name}={value}" - for name, value in [ - ("sitePackages", context.site_packages), ("cwd", context.cwd) - ] if value - ] + [ - # Runtime properties - f"-Pandroid.testInstrumentationRunnerArguments.python{name}={value}" - for name, value in [ - ("Mode", mode), ("Module", module), ("Args", join_command(context.args)) - ] if value - ] - if context.verbose >= 2: - args.append("--info") - log_verbose(context, f"> {join_command(args)}\n") - - try: - async with async_process( - *args, cwd=TESTBED_DIR, env=env, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - ) as process: - while line := (await process.stdout.readline()).decode(*DECODE_ARGS): - # Gradle may take several minutes to install SDK packages, so - # it's worth showing those messages even in non-verbose mode. - if line.startswith('Preparing "Install'): - sys.stdout.write(line) - else: - log_verbose(context, line) - - status = await wait_for(process.wait(), timeout=1) - if status == 0: - exit(0) - else: - raise CalledProcessError(status, args) - finally: - # Gradle does not stop the tests when interrupted. - if context.connected: - stop_app(context.connected) - - -async def run_testbed(context): - setup_sdk() - setup_testbed() - - if context.managed: - # In this mode, Gradle will create a device with an unpredictable name. - # So we save a list of the running devices before starting Gradle, and - # find_device then waits for a new device to appear. - initial_devices = await list_devices() - else: - # In case the previous shutdown was unclean, make sure the app isn't - # running, otherwise we might show logs from a previous run. This is - # unnecessary in --managed mode, because Gradle creates a new emulator - # every time. - stop_app(context.connected) - initial_devices = None - - try: - async with asyncio.TaskGroup() as tg: - tg.create_task(logcat_task(context, initial_devices)) - tg.create_task(gradle_task(context)) - except* MySystemExit as e: - raise SystemExit(*e.exceptions[0].args) from None - except* CalledProcessError as e: - # If Python produced no output, then the user probably wants to see the - # verbose output to explain why the test failed. - if not python_started: - for stream, line in hidden_output: - stream.write(line) - - # Extract it from the ExceptionGroup so it can be handled by `main`. - raise e.exceptions[0] - - -def package_version(prefix_dir): - patchlevel_glob = f"{prefix_dir}/include/python*/patchlevel.h" - patchlevel_paths = glob(patchlevel_glob) - if len(patchlevel_paths) != 1: - sys.exit(f"{patchlevel_glob} matched {len(patchlevel_paths)} paths.") - - for line in open(patchlevel_paths[0]): - if match := re.fullmatch(r'\s*#define\s+PY_VERSION\s+"(.+)"\s*', line): - version = match[1] - break - else: - sys.exit(f"Failed to find Python version in {patchlevel_paths[0]}.") - - # If not building against a tagged commit, add a timestamp to the version. - # Follow the PyPA version number rules, as this will make it easier to - # process with other tools. - if version.endswith("+"): - version += datetime.now(timezone.utc).strftime("%Y%m%d.%H%M%S") - - return version - - -def package(context): - prefix_dir = subdir(context.host, "prefix") - version = package_version(prefix_dir) - - with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: - temp_dir = Path(temp_dir) - - # Include all tracked files from the Android directory. - for line in run( - ["git", "ls-files"], - cwd=ANDROID_DIR, capture_output=True, text=True, log=False, - ).stdout.splitlines(): - src = ANDROID_DIR / line - dst = temp_dir / line - dst.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(src, dst, follow_symlinks=False) - - # Include anything from the prefix directory which could be useful - # either for embedding Python in an app, or building third-party - # packages against it. - for rel_dir, patterns in [ - ("include", ["openssl*", "python*", "sqlite*"]), - ("lib", ["engines-3", "libcrypto*.so", "libpython*", "libsqlite*", - "libssl*.so", "ossl-modules", "python*"]), - ("lib/pkgconfig", ["*crypto*", "*ssl*", "*python*", "*sqlite*"]), - ]: - for pattern in patterns: - for src in glob(f"{prefix_dir}/{rel_dir}/{pattern}"): - dst = temp_dir / relpath(src, prefix_dir.parent) - dst.parent.mkdir(parents=True, exist_ok=True) - if Path(src).is_dir(): - shutil.copytree( - src, dst, symlinks=True, - ignore=lambda *args: ["__pycache__"] - ) - else: - shutil.copy2(src, dst, follow_symlinks=False) - - dist_dir = subdir(context.host, "dist", create=True) - package_path = shutil.make_archive( - f"{dist_dir}/python-{version}-{context.host}", "gztar", temp_dir - ) - print(f"Wrote {package_path}") - - -def env(context): - print_env(android_env(getattr(context, "host", None))) - - -# Handle SIGTERM the same way as SIGINT. This ensures that if we're terminated -# by the buildbot worker, we'll make an attempt to clean up our subprocesses. -def install_signal_handler(): - def signal_handler(*args): - os.kill(os.getpid(), signal.SIGINT) - - signal.signal(signal.SIGTERM, signal_handler) - - -def parse_args(): - parser = argparse.ArgumentParser() - subcommands = parser.add_subparsers(dest="subcommand", required=True) - - # Subcommands - build = subcommands.add_parser( - "build", help="Run configure-build, make-build, configure-host and " - "make-host") - configure_build = subcommands.add_parser( - "configure-build", help="Run `configure` for the build Python") - subcommands.add_parser( - "make-build", help="Run `make` for the build Python") - configure_host = subcommands.add_parser( - "configure-host", help="Run `configure` for Android") - make_host = subcommands.add_parser( - "make-host", help="Run `make` for Android") - - subcommands.add_parser("clean", help="Delete all build directories") - subcommands.add_parser("build-testbed", help="Build the testbed app") - test = subcommands.add_parser("test", help="Run the testbed app") - package = subcommands.add_parser("package", help="Make a release package") - env = subcommands.add_parser("env", help="Print environment variables") - - # Common arguments - for subcommand in build, configure_build, configure_host: - subcommand.add_argument( - "--clean", action="store_true", default=False, dest="clean", - help="Delete the relevant build directories first") - - host_commands = [build, configure_host, make_host, package] - if in_source_tree: - host_commands.append(env) - for subcommand in host_commands: - subcommand.add_argument( - "host", metavar="HOST", choices=HOSTS, - help="Host triplet: choices=[%(choices)s]") - - for subcommand in build, configure_build, configure_host: - subcommand.add_argument("args", nargs="*", - help="Extra arguments to pass to `configure`") - - # Test arguments - test.add_argument( - "-v", "--verbose", action="count", default=0, - help="Show Gradle output, and non-Python logcat messages. " - "Use twice to include high-volume messages which are rarely useful.") - - device_group = test.add_mutually_exclusive_group(required=True) - device_group.add_argument( - "--connected", metavar="SERIAL", help="Run on a connected device. " - "Connect it yourself, then get its serial from `adb devices`.") - device_group.add_argument( - "--managed", metavar="NAME", help="Run on a Gradle-managed device. " - "These are defined in `managedDevices` in testbed/app/build.gradle.kts.") - - test.add_argument( - "--site-packages", metavar="DIR", type=abspath, - help="Directory to copy as the app's site-packages.") - test.add_argument( - "--cwd", metavar="DIR", type=abspath, - help="Directory to copy as the app's working directory.") - - mode_group = test.add_mutually_exclusive_group() - mode_group.add_argument( - "-c", dest="command", help="Execute the given Python code.") - mode_group.add_argument( - "-m", dest="module", help="Execute the module with the given name.") - test.epilog = ( - "If neither -c nor -m are passed, the default is '-m test', which will " - "run Python's own test suite.") - test.add_argument( - "args", nargs="*", help=f"Arguments to add to sys.argv. " - f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.") - - return parser.parse_args() - - -def main(): - install_signal_handler() - - # Under the buildbot, stdout is not a TTY, but we must still flush after - # every line to make sure our output appears in the correct order relative - # to the output of our subprocesses. - for stream in [sys.stdout, sys.stderr]: - stream.reconfigure(line_buffering=True) - - context = parse_args() - dispatch = { - "configure-build": configure_build_python, - "make-build": make_build_python, - "configure-host": configure_host_python, - "make-host": make_host_python, - "build": build_all, - "clean": clean_all, - "build-testbed": build_testbed, - "test": run_testbed, - "package": package, - "env": env, - } - - try: - result = dispatch[context.subcommand](context) - if asyncio.iscoroutine(result): - asyncio.run(result) - except CalledProcessError as e: - print_called_process_error(e) - sys.exit(1) - - -def print_called_process_error(e): - for stream_name in ["stdout", "stderr"]: - content = getattr(e, stream_name) - stream = getattr(sys, stream_name) - if content: - stream.write(content) - if not content.endswith("\n"): - stream.write("\n") - - # shlex uses single quotes, so we surround the command with double quotes. - print( - f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}' - ) - - -if __name__ == "__main__": - main() diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts deleted file mode 100644 index 92cffd61f86876e..000000000000000 --- a/Android/testbed/app/build.gradle.kts +++ /dev/null @@ -1,268 +0,0 @@ -import com.android.build.api.variant.* -import kotlin.math.max - -plugins { - id("com.android.application") - id("org.jetbrains.kotlin.android") -} - -val ANDROID_DIR = file("../..") -val PYTHON_DIR = ANDROID_DIR.parentFile!! -val PYTHON_CROSS_DIR = file("$PYTHON_DIR/cross-build") -val inSourceTree = ( - ANDROID_DIR.name == "Android" && file("$PYTHON_DIR/pyconfig.h.in").exists() -) - -val KNOWN_ABIS = mapOf( - "aarch64-linux-android" to "arm64-v8a", - "x86_64-linux-android" to "x86_64", -) - -// Discover prefixes. -val prefixes = ArrayList() -if (inSourceTree) { - for ((triplet, _) in KNOWN_ABIS.entries) { - val prefix = file("$PYTHON_CROSS_DIR/$triplet/prefix") - if (prefix.exists()) { - prefixes.add(prefix) - } - } -} else { - // Testbed is inside a release package. - val prefix = file("$ANDROID_DIR/prefix") - if (prefix.exists()) { - prefixes.add(prefix) - } -} -if (prefixes.isEmpty()) { - throw GradleException( - "No Android prefixes found: see README.md for testing instructions" - ) -} - -// Detect Python versions and ABIs. -lateinit var pythonVersion: String -var abis = HashMap() -for ((i, prefix) in prefixes.withIndex()) { - val libDir = file("$prefix/lib") - val version = run { - for (filename in libDir.list()!!) { - """python(\d+\.\d+)""".toRegex().matchEntire(filename)?.let { - return@run it.groupValues[1] - } - } - throw GradleException("Failed to find Python version in $libDir") - } - if (i == 0) { - pythonVersion = version - } else if (pythonVersion != version) { - throw GradleException( - "${prefixes[0]} is Python $pythonVersion, but $prefix is Python $version" - ) - } - - val libPythonDir = file("$libDir/python$pythonVersion") - val triplet = run { - for (filename in libPythonDir.list()!!) { - """_sysconfigdata__android_(.+).py""".toRegex().matchEntire(filename)?.let { - return@run it.groupValues[1] - } - } - throw GradleException("Failed to find Python triplet in $libPythonDir") - } - abis[prefix] = KNOWN_ABIS[triplet]!! -} - - -android { - val androidEnvFile = file("../../android-env.sh").absoluteFile - - namespace = "org.python.testbed" - compileSdk = 34 - - defaultConfig { - applicationId = "org.python.testbed" - - minSdk = androidEnvFile.useLines { - for (line in it) { - """ANDROID_API_LEVEL:=(\d+)""".toRegex().find(line)?.let { - return@useLines it.groupValues[1].toInt() - } - } - throw GradleException("Failed to find API level in $androidEnvFile") - } - targetSdk = 34 - - versionCode = 1 - versionName = "1.0" - - ndk.abiFilters.addAll(abis.values) - externalNativeBuild.cmake.arguments( - "-DPYTHON_PREFIX_DIR=" + if (inSourceTree) { - // AGP uses the ${} syntax for its own purposes, so use a Jinja style - // placeholder. - "$PYTHON_CROSS_DIR/{{triplet}}/prefix" - } else { - prefixes[0] - }, - "-DPYTHON_VERSION=$pythonVersion", - "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON", - ) - - testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" - } - - ndkVersion = androidEnvFile.useLines { - for (line in it) { - """ndk_version=(\S+)""".toRegex().find(line)?.let { - return@useLines it.groupValues[1] - } - } - throw GradleException("Failed to find NDK version in $androidEnvFile") - } - externalNativeBuild.cmake { - path("src/main/c/CMakeLists.txt") - } - - // Set this property to something non-empty, otherwise it'll use the default - // list, which ignores asset directories beginning with an underscore. - aaptOptions.ignoreAssetsPattern = ".git" - - compileOptions { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 - } - kotlinOptions { - jvmTarget = "1.8" - } - - testOptions { - managedDevices { - localDevices { - create("minVersion") { - device = "Small Phone" - - // Managed devices have a minimum API level of 27. - apiLevel = max(27, defaultConfig.minSdk!!) - - // ATD devices are smaller and faster, but have a minimum - // API level of 30. - systemImageSource = if (apiLevel >= 30) "aosp-atd" else "aosp" - } - - create("maxVersion") { - device = "Small Phone" - apiLevel = defaultConfig.targetSdk!! - systemImageSource = "aosp-atd" - } - } - - // If the previous test run succeeded and nothing has changed, - // Gradle thinks there's no need to run it again. Override that. - afterEvaluate { - (localDevices.names + listOf("connected")).forEach { - tasks.named("${it}DebugAndroidTest") { - outputs.upToDateWhen { false } - } - } - } - } - } -} - -dependencies { - implementation("androidx.appcompat:appcompat:1.6.1") - implementation("com.google.android.material:material:1.11.0") - implementation("androidx.constraintlayout:constraintlayout:2.1.4") - androidTestImplementation("androidx.test.ext:junit:1.1.5") - androidTestImplementation("androidx.test:rules:1.5.0") -} - - -// Create some custom tasks to copy Python and its standard library from -// elsewhere in the repository. -androidComponents.onVariants { variant -> - val pyPlusVer = "python$pythonVersion" - generateTask(variant, variant.sources.assets!!) { - into("python") { - // Include files such as pyconfig.h are used by some of the tests. - into("include/$pyPlusVer") { - for (prefix in prefixes) { - from("$prefix/include/$pyPlusVer") - } - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - } - - into("lib/$pyPlusVer") { - // To aid debugging, the source directory takes priority when - // running inside a CPython source tree. - if (inSourceTree) { - from("$PYTHON_DIR/Lib") - } - for (prefix in prefixes) { - from("$prefix/lib/$pyPlusVer") - } - - into("site-packages") { - from("$projectDir/src/main/python") - - val sitePackages = findProperty("python.sitePackages") as String? - if (!sitePackages.isNullOrEmpty()) { - if (!file(sitePackages).exists()) { - throw GradleException("$sitePackages does not exist") - } - from(sitePackages) - } - } - - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - exclude("**/__pycache__") - } - - into("cwd") { - val cwd = findProperty("python.cwd") as String? - if (!cwd.isNullOrEmpty()) { - if (!file(cwd).exists()) { - throw GradleException("$cwd does not exist") - } - from(cwd) - } - } - } - } - - generateTask(variant, variant.sources.jniLibs!!) { - for ((prefix, abi) in abis.entries) { - into(abi) { - from("$prefix/lib") - include("libpython*.*.so") - include("lib*_python.so") - } - } - } -} - - -fun generateTask( - variant: ApplicationVariant, directories: SourceDirectories, - configure: GenerateTask.() -> Unit -) { - val taskName = "generate" + - listOf(variant.name, "Python", directories.name) - .map { it.replaceFirstChar(Char::uppercase) } - .joinToString("") - - directories.addGeneratedSourceDirectory( - tasks.register(taskName) { - into(outputDir) - configure() - }, - GenerateTask::outputDir) -} - - -// addGeneratedSourceDirectory requires the task to have a DirectoryProperty. -abstract class GenerateTask: Sync() { - @get:OutputDirectory - abstract val outputDir: DirectoryProperty -} diff --git a/Android/testbed/app/src/main/c/main_activity.c b/Android/testbed/app/src/main/c/main_activity.c deleted file mode 100644 index ec7f93a3e5ee13a..000000000000000 --- a/Android/testbed/app/src/main/c/main_activity.c +++ /dev/null @@ -1,152 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - - -static void throw_runtime_exception(JNIEnv *env, const char *message) { - (*env)->ThrowNew( - env, - (*env)->FindClass(env, "java/lang/RuntimeException"), - message); -} - - -// --- Stdio redirection ------------------------------------------------------ - -// Most apps won't need this, because the Python-level sys.stdout and sys.stderr -// are redirected to the Android logcat by Python itself. However, in the -// testbed it's useful to redirect the native streams as well, to debug problems -// in the Python startup or redirection process. -// -// Based on -// https://github.com/beeware/briefcase-android-gradle-template/blob/v0.3.11/%7B%7B%20cookiecutter.safe_formal_name%20%7D%7D/app/src/main/cpp/native-lib.cpp - -typedef struct { - FILE *file; - int fd; - android_LogPriority priority; - char *tag; - int pipe[2]; -} StreamInfo; - -// The FILE member can't be initialized here because stdout and stderr are not -// compile-time constants. Instead, it's initialized immediately before the -// redirection. -static StreamInfo STREAMS[] = { - {NULL, STDOUT_FILENO, ANDROID_LOG_INFO, "native.stdout", {-1, -1}}, - {NULL, STDERR_FILENO, ANDROID_LOG_WARN, "native.stderr", {-1, -1}}, - {NULL, -1, ANDROID_LOG_UNKNOWN, NULL, {-1, -1}}, -}; - -// The maximum length of a log message in bytes, including the level marker and -// tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD in -// platform/system/logging/liblog/include/log/log.h. As of API level 30, messages -// longer than this will be be truncated by logcat. This limit has already been -// reduced at least once in the history of Android (from 4076 to 4068 between API -// level 23 and 26), so leave some headroom. -static const int MAX_BYTES_PER_WRITE = 4000; - -static void *redirection_thread(void *arg) { - StreamInfo *si = (StreamInfo*)arg; - ssize_t read_size; - char buf[MAX_BYTES_PER_WRITE]; - while ((read_size = read(si->pipe[0], buf, sizeof buf - 1)) > 0) { - buf[read_size] = '\0'; /* add null-terminator */ - __android_log_write(si->priority, si->tag, buf); - } - return 0; -} - -static char *redirect_stream(StreamInfo *si) { - /* make the FILE unbuffered, to ensure messages are never lost */ - if (setvbuf(si->file, 0, _IONBF, 0)) { - return "setvbuf"; - } - - /* create the pipe and redirect the file descriptor */ - if (pipe(si->pipe)) { - return "pipe"; - } - if (dup2(si->pipe[1], si->fd) == -1) { - return "dup2"; - } - - /* start the logging thread */ - pthread_t thr; - if ((errno = pthread_create(&thr, 0, redirection_thread, si))) { - return "pthread_create"; - } - if ((errno = pthread_detach(thr))) { - return "pthread_detach"; - } - return 0; -} - -JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToLogcat( - JNIEnv *env, jobject obj -) { - STREAMS[0].file = stdout; - STREAMS[1].file = stderr; - for (StreamInfo *si = STREAMS; si->file; si++) { - char *error_prefix; - if ((error_prefix = redirect_stream(si))) { - char error_message[1024]; - snprintf(error_message, sizeof(error_message), - "%s: %s", error_prefix, strerror(errno)); - throw_runtime_exception(env, error_message); - return; - } - } -} - - -// --- Python initialization --------------------------------------------------- - -static PyStatus set_config_string( - JNIEnv *env, PyConfig *config, wchar_t **config_str, jstring value -) { - const char *value_utf8 = (*env)->GetStringUTFChars(env, value, NULL); - PyStatus status = PyConfig_SetBytesString(config, config_str, value_utf8); - (*env)->ReleaseStringUTFChars(env, value, value_utf8); - return status; -} - -static void throw_status(JNIEnv *env, PyStatus status) { - throw_runtime_exception(env, status.err_msg ? status.err_msg : ""); -} - -JNIEXPORT int JNICALL Java_org_python_testbed_PythonTestRunner_runPython( - JNIEnv *env, jobject obj, jstring home, jstring runModule -) { - PyConfig config; - PyStatus status; - PyConfig_InitIsolatedConfig(&config); - - status = set_config_string(env, &config, &config.home, home); - if (PyStatus_Exception(status)) { - throw_status(env, status); - return 1; - } - - status = set_config_string(env, &config, &config.run_module, runModule); - if (PyStatus_Exception(status)) { - throw_status(env, status); - return 1; - } - - // Some tests generate SIGPIPE and SIGXFSZ, which should be ignored. - config.install_signal_handlers = 1; - - status = Py_InitializeFromConfig(&config); - if (PyStatus_Exception(status)) { - throw_status(env, status); - return 1; - } - - return Py_RunMain(); -} diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt deleted file mode 100644 index ef28948486fb529..000000000000000 --- a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt +++ /dev/null @@ -1,92 +0,0 @@ -package org.python.testbed - -import android.content.Context -import android.os.* -import android.system.Os -import android.widget.TextView -import androidx.appcompat.app.* -import java.io.* - - -// Launching the tests from an activity is OK for a quick check, but for -// anything more complicated it'll be more convenient to use `android.py test` -// to launch the tests via PythonSuite. -class MainActivity : AppCompatActivity() { - override fun onCreate(savedInstanceState: Bundle?) { - super.onCreate(savedInstanceState) - setContentView(R.layout.activity_main) - val status = PythonTestRunner(this).run("-m", "test", "-W -uall") - findViewById(R.id.tvHello).text = "Exit status $status" - } -} - - -class PythonTestRunner(val context: Context) { - fun run(instrumentationArgs: Bundle) = run( - instrumentationArgs.getString("pythonMode")!!, - instrumentationArgs.getString("pythonModule")!!, - instrumentationArgs.getString("pythonArgs") ?: "", - ) - - /** Run Python. - * - * @param mode Either "-c" or "-m". - * @param module Python statements for "-c" mode, or a module name for - * "-m" mode. - * @param args Arguments to add to sys.argv. Will be parsed by `shlex.split`. - * @return The Python exit status: zero on success, nonzero on failure. */ - fun run(mode: String, module: String, args: String) : Int { - Os.setenv("PYTHON_MODE", mode, true) - Os.setenv("PYTHON_MODULE", module, true) - Os.setenv("PYTHON_ARGS", args, true) - - // Python needs this variable to help it find the temporary directory, - // but Android only sets it on API level 33 and later. - Os.setenv("TMPDIR", context.cacheDir.toString(), false) - - val pythonHome = extractAssets() - System.loadLibrary("main_activity") - redirectStdioToLogcat() - - // The main module is in src/main/python. We don't simply call it - // "main", as that could clash with third-party test code. - return runPython(pythonHome.toString(), "android_testbed_main") - } - - private fun extractAssets() : File { - val pythonHome = File(context.filesDir, "python") - if (pythonHome.exists() && !pythonHome.deleteRecursively()) { - throw RuntimeException("Failed to delete $pythonHome") - } - extractAssetDir("python", context.filesDir) - return pythonHome - } - - private fun extractAssetDir(path: String, targetDir: File) { - val names = context.assets.list(path) - ?: throw RuntimeException("Failed to list $path") - val targetSubdir = File(targetDir, path) - if (!targetSubdir.mkdirs()) { - throw RuntimeException("Failed to create $targetSubdir") - } - - for (name in names) { - val subPath = "$path/$name" - val input: InputStream - try { - input = context.assets.open(subPath) - } catch (e: FileNotFoundException) { - extractAssetDir(subPath, targetDir) - continue - } - input.use { - File(targetSubdir, name).outputStream().use { output -> - input.copyTo(output) - } - } - } - } - - private external fun redirectStdioToLogcat() - private external fun runPython(home: String, runModule: String) : Int -} diff --git a/Android/testbed/app/src/main/python/android_testbed_main.py b/Android/testbed/app/src/main/python/android_testbed_main.py deleted file mode 100644 index 31b8e5343a84497..000000000000000 --- a/Android/testbed/app/src/main/python/android_testbed_main.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -import runpy -import shlex -import signal -import sys - -# Some tests use SIGUSR1, but that's blocked by default in an Android app in -# order to make it available to `sigwait` in the Signal Catcher thread. -# (https://cs.android.com/android/platform/superproject/+/android14-qpr3-release:art/runtime/signal_catcher.cc). -# That thread's functionality is only useful for debugging the JVM, so disabling -# it should not weaken the tests. -# -# There's no safe way of stopping the thread completely (#123982), but simply -# unblocking SIGUSR1 is enough to fix most tests. -# -# However, in tests that generate multiple different signals in quick -# succession, it's possible for SIGUSR1 to arrive while the main thread is busy -# running the C-level handler for a different signal. In that case, the SIGUSR1 -# may be sent to the Signal Catcher thread instead, which will generate a log -# message containing the text "reacting to signal". -# -# Such tests may need to be changed in one of the following ways: -# * Use a signal other than SIGUSR1 (e.g. test_stress_delivery_simultaneous in -# test_signal.py). -# * Send the signal to a specific thread rather than the whole process (e.g. -# test_signals in test_threadsignals.py. -signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1]) - -mode = os.environ["PYTHON_MODE"] -module = os.environ["PYTHON_MODULE"] -sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"]) - -cwd = f"{sys.prefix}/cwd" -if not os.path.exists(cwd): - # Empty directories are lost in the asset packing/unpacking process. - os.mkdir(cwd) -os.chdir(cwd) - -if mode == "-c": - # In -c mode, sys.path starts with an empty string, which means whatever the current - # working directory is at the moment of each import. - sys.path.insert(0, "") - exec(module, {}) -elif mode == "-m": - sys.path.insert(0, os.getcwd()) - runpy.run_module(module, run_name="__main__", alter_sys=True) -else: - raise ValueError(f"unknown mode: {mode}") diff --git a/Doc/.ruff.toml b/Doc/.ruff.toml index 3e676e13c3f41ac..6b573fd58d089bc 100644 --- a/Doc/.ruff.toml +++ b/Doc/.ruff.toml @@ -32,6 +32,9 @@ ignore = [ "E501", # Ignore line length errors (we use auto-formatting) ] +[lint.per-file-ignores] +"tools/check-html-ids.py" = ["I001"] # Unsorted imports + [format] preview = true quote-style = "preserve" diff --git a/Doc/Makefile b/Doc/Makefile index 84578c5c57f478f..d77ece1e681bcfc 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -13,7 +13,7 @@ JOBS = auto PAPER = SOURCES = DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py) -REQUIREMENTS = requirements.txt +REQUIREMENTS = pylock.toml SPHINXERRORHANDLING = --fail-on-warning # Internal variables. @@ -58,7 +58,7 @@ build: @if [ -f ../Misc/NEWS ] ; then \ echo "Using existing Misc/NEWS file"; \ cp ../Misc/NEWS build/NEWS; \ - elif $(BLURB) help >/dev/null 2>&1 && $(SPHINXBUILD) --version >/dev/null 2>&1; then \ + elif $(BLURB) --version && $(SPHINXBUILD) --version ; then \ if [ -d ../Misc/NEWS.d ]; then \ echo "Building NEWS from Misc/NEWS.d with blurb"; \ $(BLURB) merge -f build/NEWS; \ @@ -89,7 +89,8 @@ htmlhelp: build .PHONY: latex latex: BUILDER = latex -latex: build +latex: _ensure-sphinxcontrib-svg2pdfconverter + $(MAKE) build BUILDER=$(BUILDER) @echo "Build finished; the LaTeX files are in build/latex." @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ "run these through (pdf)latex." @@ -140,7 +141,8 @@ doctest: pydoc-topics: BUILDER = pydoc-topics pydoc-topics: build @echo "Building finished; now run this:" \ - "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" + "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" \ + "&& cp build/pydoc-topics/module_docs.py ../Lib/pydoc_data/module_docs.py" .PHONY: gettext gettext: BUILDER = gettext @@ -184,7 +186,7 @@ venv: fi .PHONY: dist-no-html -dist-no-html: dist-text dist-pdf dist-epub dist-texinfo +dist-no-html: dist-text dist-epub dist-texinfo .PHONY: dist dist: @@ -230,7 +232,7 @@ dist-text: @echo "Build finished and archived!" .PHONY: dist-pdf -dist-pdf: +dist-pdf: _ensure-sphinxcontrib-svg2pdfconverter # archive the A4 latex @echo "Building LaTeX (A4 paper)..." mkdir -p dist @@ -241,7 +243,8 @@ dist-pdf: # as otherwise the full latexmk process is run twice. # ($$ is needed to escape the $; https://www.gnu.org/software/make/manual/make.html#Basics-of-Variable-References) -sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile - (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) + if [ -n "$(filter output-sync,$(value .FEATURES))" ]; then OUTPUTSYNC=--output-sync; else OUTPUTSYNC=; fi && \ + (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`getconf _NPROCESSORS_ONLN`+1)) $$OUTPUTSYNC LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-a4.zip cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2 @echo "Build finished and archived!" @@ -290,6 +293,10 @@ _ensure-pre-commit: _ensure-sphinx-autobuild: $(MAKE) _ensure-package PACKAGE=sphinx-autobuild +.PHONY: _ensure-sphinxcontrib-svg2pdfconverter +_ensure-sphinxcontrib-svg2pdfconverter: + $(MAKE) _ensure-package PACKAGE=sphinxcontrib-svg2pdfconverter + .PHONY: check check: _ensure-pre-commit $(VENVDIR)/bin/python3 -m pre_commit run --all-files @@ -334,3 +341,9 @@ autobuild-stable-html: exit 1;; \ esac @$(MAKE) autobuild-dev-html + +# Collect HTML IDs to a JSON document +.PHONY: html-ids +html-ids: + $(PYTHON) tools/check-html-ids.py collect build/html \ + -o build/html/html-ids.json.gz diff --git a/Doc/_static/profiling-sampling-visualization.css b/Doc/_static/profiling-sampling-visualization.css new file mode 100644 index 000000000000000..6bfbec3b8a60449 --- /dev/null +++ b/Doc/_static/profiling-sampling-visualization.css @@ -0,0 +1,570 @@ +/** + * Sampling Profiler Visualization - Scoped CSS + */ + +.sampling-profiler-viz { + /* Match docs background colors */ + --bg-page: #ffffff; + --bg-panel: #ffffff; + --bg-subtle: #f8f8f8; + --bg-code: #f8f8f8; + + /* Match docs border style */ + --border-color: #e1e4e8; + --border-accent: #3776ab; + + /* Match docs text colors */ + --text-primary: #0d0d0d; + --text-secondary: #505050; + --text-muted: #6e6e6e; + --text-code: #333333; + + /* Accent colors */ + --color-python-blue: #306998; + --color-green: #388e3c; + --color-orange: #e65100; + --color-purple: #7b1fa2; + --color-red: #c62828; + --color-teal: #00897b; + --color-yellow: #d4a910; + --color-highlight: #fff9e6; + + --radius-lg: 8px; + --radius-md: 6px; + --radius-sm: 4px; + + /* Lighter shadows to match docs style */ + --shadow-card: 0 1px 3px rgba(0, 0, 0, 0.08); + --shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.04); + + --container-height: 520px; + --code-panel-width: 320px; + + /* Reset for isolation */ + font-family: var(--font-ui); + line-height: 1.5; + font-weight: 400; + color: var(--text-primary); + background-color: var(--bg-page); + font-synthesis: none; + text-rendering: optimizeLegibility; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + + /* Layout */ + position: relative; + width: 100%; + max-width: 920px; + height: var(--container-height); + display: grid; + grid-template-columns: var(--code-panel-width) 1fr; + margin: 24px auto; + border-radius: var(--radius-lg); + overflow: hidden; + box-shadow: var(--shadow-card); + border: 1px solid var(--border-color); + background: var(--bg-panel); + /* Prevent any DOM changes inside from affecting page scroll */ + contain: strict; +} + +.sampling-profiler-viz * { + box-sizing: border-box; +} + +/* Code Panel - Left Column */ +.sampling-profiler-viz #code-panel { + background: var(--bg-panel); + border-right: 1px solid var(--border-color); + overflow-y: auto; + font-family: var(--font-mono); + font-size: 12px; + line-height: 1.6; + display: flex; + flex-direction: column; +} + +.sampling-profiler-viz #code-panel .code-panel-title { + padding: 12px 16px; + font-size: 10px; + font-weight: 600; + color: var(--text-muted); + border-bottom: 1px solid var(--border-color); + background: var(--bg-code); + text-transform: uppercase; + letter-spacing: 1px; + flex-shrink: 0; +} + +.sampling-profiler-viz #code-panel .code-container { + margin: 0; + padding: 12px 0; + overflow-x: auto; + flex: 1; +} + +.sampling-profiler-viz #code-panel .line { + display: flex; + padding: 1px 0; + min-height: 20px; + transition: background-color 0.1s ease; +} + +.sampling-profiler-viz #code-panel .line-number { + color: var(--text-muted); + min-width: 40px; + text-align: right; + padding-right: 12px; + padding-left: 12px; + user-select: none; + flex-shrink: 0; + font-size: 11px; +} + +.sampling-profiler-viz #code-panel .line-content { + flex: 1; + color: var(--text-code); + padding-right: 12px; + white-space: pre; +} + +.sampling-profiler-viz #code-panel .line.highlighted { + background: var(--color-highlight); + border-left: 3px solid var(--color-yellow); +} + +.sampling-profiler-viz #code-panel .line.highlighted .line-number { + color: var(--color-yellow); + padding-left: 9px; +} + +.sampling-profiler-viz #code-panel .line.highlighted .line-content { + font-weight: 600; +} + +/* Python Syntax Highlighting */ +.sampling-profiler-viz #code-panel .keyword { + color: var(--color-red); + font-weight: 600; +} + +.sampling-profiler-viz #code-panel .function { + color: var(--color-purple); + font-weight: 600; +} + +.sampling-profiler-viz #code-panel .number { + color: var(--color-python-blue); +} + +.sampling-profiler-viz #code-panel .string { + color: #032f62; +} + +.sampling-profiler-viz #code-panel .comment { + color: #6a737d; + font-style: italic; +} + +.sampling-profiler-viz #code-panel .builtin { + color: var(--color-python-blue); +} + +/* Visualization Column - Right Side */ +.sampling-profiler-viz .viz-column { + display: flex; + flex-direction: column; + background: var(--bg-subtle); + overflow: hidden; +} + +/* Stack Section */ +.sampling-profiler-viz .stack-section { + padding: 12px 16px; + flex: 1; + min-height: 150px; + overflow-y: auto; +} + +.sampling-profiler-viz .stack-section-title { + font-size: 10px; + font-weight: 600; + color: var(--text-muted); + text-transform: uppercase; + letter-spacing: 1px; + margin-bottom: 10px; +} + +.sampling-profiler-viz .stack-visualization { + display: flex; + flex-direction: column; + gap: 4px; + min-height: 80px; +} + +/* Stack Frames - Vertical Layout */ +.sampling-profiler-viz .stack-frame { + position: relative; + width: 100%; + height: 32px; + cursor: pointer; + contain: layout style paint; + opacity: 0; + transform: translateY(-10px); +} + +.sampling-profiler-viz .stack-frame.visible { + opacity: 1; + transform: translateY(0); + transition: + opacity 0.3s ease, + transform 0.3s ease; +} + +.sampling-profiler-viz .stack-frame-bg { + position: absolute; + inset: 0; + border-radius: var(--radius-sm); + transition: opacity 0.15s; +} + +.sampling-profiler-viz .stack-frame-text { + position: absolute; + left: 10px; + top: 50%; + transform: translateY(-50%); + font: 500 11px var(--font-mono); + color: white; + pointer-events: none; +} + +.sampling-profiler-viz .stack-frame-flash { + position: absolute; + inset: 0; + background: white; + border-radius: var(--radius-sm); + opacity: 0; + pointer-events: none; +} + +.sampling-profiler-viz .stack-frame:hover .stack-frame-bg { + opacity: 0.85; +} + +/* Flying frames */ +.sampling-profiler-viz .stack-frame.flying { + pointer-events: none; + z-index: 1000; + position: fixed; + top: 0; + left: 0; + width: auto; + height: 32px; + opacity: 1; +} + +/* Flying stack clone */ +.stack-visualization.flying-clone { + transform-origin: center center; + will-change: transform, opacity; +} + +/* Sampling Panel */ +.sampling-profiler-viz .sampling-panel { + margin: 0 16px 12px 16px; + background: var(--bg-panel); + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + box-shadow: var(--shadow-sm); + overflow: hidden; + display: flex; + flex-direction: column; + flex: 0 0 auto; + height: 200px; + /* Lock font size to prevent Sphinx responsive scaling */ + font-size: 12px; +} + +.sampling-profiler-viz .sampling-header { + padding: 8px 10px; + border-bottom: 1px solid var(--border-color); + flex-shrink: 0; +} + +.sampling-profiler-viz .sampling-title { + font: 600 10px var(--font-mono); + color: var(--text-primary); + margin: 0 0 3px 0; +} + +.sampling-profiler-viz .sampling-stats { + font: 400 9px var(--font-mono); + color: var(--text-secondary); + display: flex; + gap: 12px; +} + +.sampling-profiler-viz .sampling-stats .missed { + color: var(--color-red); +} + +.sampling-profiler-viz .sampling-bars { + flex: 1; + padding: 10px 12px; + overflow-y: auto; +} + +.sampling-profiler-viz .sampling-bar-row { + display: flex; + align-items: center; + height: 22px; + gap: 8px; +} + +.sampling-profiler-viz .bar-label { + font: 500 8px var(--font-mono); + color: var(--text-primary); + flex-shrink: 0; + width: 60px; + overflow: hidden; + text-overflow: ellipsis; +} + +.sampling-profiler-viz .bar-container { + flex: 1; + height: 12px; + background: var(--border-color); + border-radius: 3px; + position: relative; + overflow: hidden; +} + +.sampling-profiler-viz .bar-fill { + height: 100%; + border-radius: 3px; + transition: width 0.2s ease; + min-width: 2px; +} + +.sampling-profiler-viz .bar-percent { + font: 500 8px var(--font-mono); + color: var(--text-secondary); + width: 36px; + text-align: right; + flex-shrink: 0; +} + +/* Impact effect circle */ +.impact-circle { + position: fixed; + width: 30px; + height: 30px; + border-radius: 50%; + background: var(--color-teal); + transform: translate(-50%, -50%); + pointer-events: none; + z-index: 2000; +} + +/* Control Panel - Integrated */ +.sampling-profiler-viz #control-panel { + display: flex; + align-items: center; + gap: 12px; + padding: 12px 16px; + background: var(--bg-panel); + border-top: 1px solid var(--border-color); + flex-shrink: 0; + flex-wrap: wrap; +} + +.sampling-profiler-viz .control-group { + display: flex; + gap: 6px; + align-items: center; +} + +.sampling-profiler-viz .control-group label { + font-size: 10px; + color: var(--text-muted); + font-weight: 500; + white-space: nowrap; +} + +.sampling-profiler-viz .control-btn { + background: var(--bg-panel); + color: var(--text-primary); + border: 1px solid var(--border-color); + padding: 6px 10px; + border-radius: var(--radius-sm); + cursor: pointer; + transition: all 0.15s ease; + font-family: var(--font-mono); + font-size: 11px; + font-weight: 500; + display: flex; + align-items: center; + gap: 4px; +} + +.sampling-profiler-viz .control-btn:hover { + background: var(--bg-subtle); + border-color: var(--text-muted); +} + +.sampling-profiler-viz .control-btn:active { + transform: scale(0.98); +} + +.sampling-profiler-viz .control-btn.active { + background: var(--color-python-blue); + color: white; + border-color: var(--color-python-blue); +} + +.sampling-profiler-viz .control-btn.active:hover { + background: #2f6493; +} + +/* Timeline Scrubber */ +.sampling-profiler-viz .timeline-scrubber { + flex: 1; + display: flex; + align-items: center; + gap: 8px; + min-width: 160px; +} + +.sampling-profiler-viz #timeline-scrubber { + flex: 1; + height: 5px; + border-radius: 3px; + background: var(--border-color); + outline: none; + appearance: none; + -webkit-appearance: none; + cursor: pointer; + min-width: 60px; +} + +.sampling-profiler-viz #timeline-scrubber::-webkit-slider-thumb { + -webkit-appearance: none; + width: 14px; + height: 14px; + border-radius: 50%; + background: var(--color-python-blue); + cursor: pointer; + transition: transform 0.15s; + box-shadow: 0 2px 6px rgba(0, 0, 0, 0.2); +} + +.sampling-profiler-viz #timeline-scrubber::-webkit-slider-thumb:hover { + transform: scale(1.15); +} + +.sampling-profiler-viz #timeline-scrubber::-moz-range-thumb { + width: 14px; + height: 14px; + border-radius: 50%; + background: var(--color-python-blue); + cursor: pointer; + border: none; + box-shadow: 0 2px 6px rgba(0, 0, 0, 0.2); +} + +.sampling-profiler-viz #time-display { + font: 500 10px var(--font-mono); + color: var(--text-secondary); + min-width: 90px; + text-align: right; + font-variant-numeric: tabular-nums; +} + +/* Sample Interval Slider */ +.sampling-profiler-viz #sample-interval { + width: 80px; + height: 4px; + border-radius: 2px; + background: var(--border-color); + outline: none; + appearance: none; + -webkit-appearance: none; + cursor: pointer; + flex-shrink: 0; +} + +.sampling-profiler-viz #sample-interval::-webkit-slider-thumb { + -webkit-appearance: none; + width: 12px; + height: 12px; + border-radius: 50%; + background: var(--color-teal); + cursor: pointer; + transition: transform 0.15s; + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2); +} + +.sampling-profiler-viz #sample-interval::-webkit-slider-thumb:hover { + transform: scale(1.15); +} + +.sampling-profiler-viz #sample-interval::-moz-range-thumb { + width: 12px; + height: 12px; + border-radius: 50%; + background: var(--color-teal); + cursor: pointer; + border: none; +} + +.sampling-profiler-viz #interval-display { + font: 500 9px var(--font-mono); + color: var(--text-secondary); + min-width: 40px; + font-variant-numeric: tabular-nums; +} + +/* Flash overlay */ +.sampling-profiler-viz .flash-overlay { + position: absolute; + inset: 0; + background: white; + pointer-events: none; + opacity: 0; +} + +/* Performance optimizations */ +.sampling-profiler-viz .stack-frame, +.sampling-profiler-viz .flying-frame, +.sampling-profiler-viz .sampling-bar-row { + will-change: transform, opacity; + contain: layout style paint; +} + +/* Reduced motion support */ +@media (prefers-reduced-motion: reduce) { + .sampling-profiler-viz .stack-frame, + .sampling-profiler-viz .flying-frame, + .sampling-profiler-viz .sampling-bar-row, + .impact-circle { + animation-duration: 0.01ms !important; + transition-duration: 0.01ms !important; + } +} + +/* Responsive adjustments for narrower viewports */ +@media (max-width: 800px) { + .sampling-profiler-viz { + grid-template-columns: 280px 1fr; + --container-height: 550px; + } + + .sampling-profiler-viz #code-panel { + font-size: 11px; + } + + .sampling-profiler-viz .control-btn { + padding: 5px 8px; + font-size: 10px; + } +} diff --git a/Doc/_static/profiling-sampling-visualization.js b/Doc/_static/profiling-sampling-visualization.js new file mode 100644 index 000000000000000..3729be6795c1c82 --- /dev/null +++ b/Doc/_static/profiling-sampling-visualization.js @@ -0,0 +1,1163 @@ +/** + * Sampling Profiler Visualization + */ +(function () { + "use strict"; + + // ============================================================================ + // Configuration + // ============================================================================ + + const TIMINGS = { + sampleIntervalMin: 100, + sampleIntervalMax: 500, + sampleIntervalDefault: 200, + sampleToFlame: 600, + defaultSpeed: 0.05, + }; + + const LAYOUT = { frameSpacing: 6 }; + + // Function name to color mapping + const FUNCTION_COLORS = { + main: "#306998", + fibonacci: "#D4A910", + add: "#E65100", + multiply: "#7B1FA2", + calculate: "#D4A910", + }; + const DEFAULT_FUNCTION_COLOR = "#306998"; + + // Easing functions - cubic-bezier approximations + const EASING_MAP = { + linear: "linear", + easeOutQuad: "cubic-bezier(0.25, 0.46, 0.45, 0.94)", + easeOutCubic: "cubic-bezier(0.215, 0.61, 0.355, 1)", + }; + + function getFunctionColor(funcName) { + return FUNCTION_COLORS[funcName] || DEFAULT_FUNCTION_COLOR; + } + + // ============================================================================ + // Animation Manager + // ============================================================================ + + class AnimationManager { + constructor() { + this.activeAnimations = new Set(); + } + + to(element, props, duration, easing = "easeOutQuad", onComplete = null) { + this.killAnimationsOf(element); + + const cssEasing = EASING_MAP[easing] || EASING_MAP.easeOutQuad; + + const transformProps = {}; + const otherProps = {}; + + for (const [key, value] of Object.entries(props)) { + if (key === "position") { + if (typeof value.x === "number") transformProps.x = value.x; + if (typeof value.y === "number") transformProps.y = value.y; + } else if (key === "x" || key === "y") { + transformProps[key] = value; + } else if (key === "scale") { + transformProps.scale = value; + } else if (key === "alpha" || key === "opacity") { + otherProps.opacity = value; + } else { + otherProps[key] = value; + } + } + + const computedStyle = getComputedStyle(element); + const matrix = new DOMMatrix(computedStyle.transform); + const currentScale = Math.sqrt( + matrix.m11 * matrix.m11 + matrix.m21 * matrix.m21, + ); + + transformProps.x ??= matrix.m41; + transformProps.y ??= matrix.m42; + transformProps.scale ??= currentScale; + + const initialTransform = this._buildTransformString( + matrix.m41, + matrix.m42, + currentScale, + ); + + const finalTransform = this._buildTransformString( + transformProps.x, + transformProps.y, + transformProps.scale, + ); + + const initialKeyframe = { transform: initialTransform }; + const finalKeyframe = { transform: finalTransform }; + + for (const [key, value] of Object.entries(otherProps)) { + const currentVal = + key === "opacity" + ? element.style.opacity || computedStyle.opacity + : element.style[key]; + initialKeyframe[key] = currentVal; + finalKeyframe[key] = value; + } + + const animation = element.animate([initialKeyframe, finalKeyframe], { + duration, + easing: cssEasing, + fill: "forwards", + }); + + this.activeAnimations.add(animation); + animation.onfinish = () => { + this.activeAnimations.delete(animation); + element.style.transform = finalTransform; + for (const [key, value] of Object.entries(finalKeyframe)) { + if (key !== "transform") { + element.style[key] = typeof value === "number" ? `${value}` : value; + } + } + if (onComplete) onComplete(); + }; + + return animation; + } + + killAnimationsOf(element) { + element.getAnimations().forEach((animation) => animation.cancel()); + this.activeAnimations.forEach((animation) => { + if (animation.effect && animation.effect.target === element) { + animation.cancel(); + this.activeAnimations.delete(animation); + } + }); + } + + _buildTransformString(x, y, scale = 1) { + return `translate(${x}px, ${y}px) scale(${scale})`; + } + } + + const anim = new AnimationManager(); + + // ============================================================================ + // Execution Trace Model + // ============================================================================ + + class ExecutionEvent { + constructor( + type, + functionName, + lineno, + timestamp, + args = null, + value = null, + ) { + this.type = type; + this.functionName = functionName; + this.lineno = lineno; + this.timestamp = timestamp; + this.args = args; + this.value = value; + } + } + + class ExecutionTrace { + constructor(source, events) { + this.source = source; + this.events = events.map( + (e) => + new ExecutionEvent(e.type, e.func, e.line, e.ts, e.args, e.value), + ); + this.duration = events.length > 0 ? events[events.length - 1].ts : 0; + } + + getEventsUntil(timestamp) { + return this.events.filter((e) => e.timestamp <= timestamp); + } + + getStackAt(timestamp) { + const stack = []; + const events = this.getEventsUntil(timestamp); + + for (const event of events) { + if (event.type === "call") { + stack.push({ + func: event.functionName, + line: event.lineno, + args: event.args, + }); + } else if (event.type === "return") { + stack.pop(); + } else if (event.type === "line") { + if (stack.length > 0) { + stack[stack.length - 1].line = event.lineno; + } + } + } + return stack; + } + + getNextEvent(timestamp) { + return this.events.find((e) => e.timestamp > timestamp); + } + + getSourceLines() { + return this.source.split("\n"); + } + } + + // ============================================================================ + // Demo Data + // ============================================================================ + + // This placeholder is replaced by the profiling_trace Sphinx extension + // during the documentation build with dynamically generated trace data. + const DEMO_SIMPLE = /* PROFILING_TRACE_DATA */ null; + + // ============================================================================ + // Code Panel Component + // ============================================================================ + + class CodePanel { + constructor(source) { + this.source = source; + this.currentLine = null; + + this.element = document.createElement("div"); + this.element.id = "code-panel"; + + const title = document.createElement("div"); + title.className = "code-panel-title"; + title.textContent = "source code"; + this.element.appendChild(title); + + this.codeContainer = document.createElement("pre"); + this.codeContainer.className = "code-container"; + this.element.appendChild(this.codeContainer); + + this._renderSource(); + } + + updateSource(source) { + this.source = source; + this.codeContainer.innerHTML = ""; + this._renderSource(); + this.currentLine = null; + } + + _renderSource() { + const lines = this.source.split("\n"); + + lines.forEach((line, index) => { + const lineNumber = index + 1; + const lineDiv = document.createElement("div"); + lineDiv.className = "line"; + lineDiv.dataset.line = lineNumber; + + const lineNumSpan = document.createElement("span"); + lineNumSpan.className = "line-number"; + lineNumSpan.textContent = lineNumber; + lineDiv.appendChild(lineNumSpan); + + const codeSpan = document.createElement("span"); + codeSpan.className = "line-content"; + codeSpan.innerHTML = this._highlightSyntax(line); + lineDiv.appendChild(codeSpan); + + this.codeContainer.appendChild(lineDiv); + }); + } + + _highlightSyntax(line) { + return line + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/(f?"[^"]*"|f?'[^']*')/g, '$1') + .replace(/(#.*$)/g, '$1') + .replace( + /\b(def|if|elif|else|return|for|in|range|print|__name__|__main__)\b/g, + '$1', + ) + .replace( + /def<\/span>\s+(\w+)/g, + 'def $1', + ) + .replace(/\b(\d+)\b/g, '$1'); + } + + highlightLine(lineNumber) { + if (this.currentLine === lineNumber) return; + + if (this.currentLine !== null) { + const prevLine = this.codeContainer.querySelector( + `[data-line="${this.currentLine}"]`, + ); + if (prevLine) prevLine.classList.remove("highlighted"); + } + + if (lineNumber === null || lineNumber === undefined) { + this.currentLine = null; + return; + } + + this.currentLine = lineNumber; + const newLine = this.codeContainer.querySelector( + `[data-line="${lineNumber}"]`, + ); + if (newLine) { + newLine.classList.add("highlighted"); + } + } + + reset() { + this.highlightLine(null); + this.codeContainer.scrollTop = 0; + } + + destroy() { + this.element.remove(); + } + } + + // ============================================================================ + // Stack Frame Component + // ============================================================================ + + class DOMStackFrame { + constructor(functionName, lineno, args = null) { + this.functionName = functionName; + this.lineno = lineno; + this.args = args; + this.isActive = false; + this.color = getFunctionColor(functionName); + + this.element = document.createElement("div"); + this.element.className = "stack-frame"; + this.element.dataset.function = functionName; + + this.bgElement = document.createElement("div"); + this.bgElement.className = "stack-frame-bg"; + this.bgElement.style.backgroundColor = this.color; + this.element.appendChild(this.bgElement); + + this.textElement = document.createElement("span"); + this.textElement.className = "stack-frame-text"; + this.textElement.textContent = functionName; + this.element.appendChild(this.textElement); + + this.flashElement = document.createElement("div"); + this.flashElement.className = "stack-frame-flash"; + this.element.appendChild(this.flashElement); + + this.element.addEventListener("pointerover", this._onHover.bind(this)); + this.element.addEventListener("pointerout", this._onHoverOut.bind(this)); + } + + destroy() { + this.element.parentNode?.removeChild(this.element); + } + + updateLine(lineno) { + this.lineno = lineno; + this.textElement.textContent = this.functionName; + } + + setActive(isActive) { + if (this.isActive === isActive) return; + this.isActive = isActive; + this.bgElement.style.opacity = isActive ? "1.0" : "0.9"; + } + + _onHover() { + this.bgElement.style.opacity = "0.8"; + } + + _onHoverOut() { + this.bgElement.style.opacity = this.isActive ? "1.0" : "0.9"; + } + + flash(duration = 150) { + this.flashElement.animate([{ opacity: 1 }, { opacity: 0 }], { + duration, + easing: "ease-out", + }); + } + + getPosition() { + const rect = this.element.getBoundingClientRect(); + return { x: rect.left, y: rect.top }; + } + } + + // ============================================================================ + // Stack Visualization Component + // ============================================================================ + + class DOMStackVisualization { + constructor() { + this.frames = []; + this.frameSpacing = LAYOUT.frameSpacing; + + this.element = document.createElement("div"); + this.element.className = "stack-visualization"; + } + + processEvent(event) { + if (event.type === "call") { + this.pushFrame(event.functionName, event.lineno, event.args); + } else if (event.type === "return") { + this.popFrame(); + } else if (event.type === "line") { + this.updateTopFrameLine(event.lineno); + } + } + + updateTopFrameLine(lineno) { + if (this.frames.length > 0) { + this.frames[this.frames.length - 1].updateLine(lineno); + } + } + + pushFrame(functionName, lineno, args = null) { + if (this.frames.length > 0) { + this.frames[this.frames.length - 1].setActive(false); + } + + const frame = new DOMStackFrame(functionName, lineno, args); + frame.setActive(true); + this.element.appendChild(frame.element); + this.frames.push(frame); + + requestAnimationFrame(() => { + frame.element.classList.add("visible"); + }); + } + + popFrame() { + if (this.frames.length === 0) return; + + const frame = this.frames.pop(); + frame.element.classList.remove("visible"); + setTimeout(() => frame.destroy(), 300); + + if (this.frames.length > 0) { + this.frames[this.frames.length - 1].setActive(true); + } + } + + clear() { + this.frames.forEach((frame) => frame.destroy()); + this.frames = []; + this.element.innerHTML = ""; + } + + flashAll() { + this.frames.forEach((frame) => frame.flash()); + } + + createStackClone(container) { + const clone = this.element.cloneNode(false); + clone.className = "stack-visualization flying-clone"; + + const elementRect = this.element.getBoundingClientRect(); + const containerRect = container.getBoundingClientRect(); + + // Position relative to container since contain: strict makes position:fixed relative to container + clone.style.position = "absolute"; + clone.style.left = elementRect.left - containerRect.left + "px"; + clone.style.top = elementRect.top - containerRect.top + "px"; + clone.style.width = elementRect.width + "px"; + clone.style.pointerEvents = "none"; + clone.style.zIndex = "1000"; + + this.frames.forEach((frame) => { + const frameClone = frame.element.cloneNode(true); + frameClone.classList.add("visible"); + frameClone.style.opacity = "1"; + frameClone.style.transform = "translateY(0)"; + frameClone.style.transition = "none"; + clone.appendChild(frameClone); + }); + + container.appendChild(clone); + return clone; + } + + updateToMatch(targetStack) { + while (this.frames.length > targetStack.length) { + this.popFrame(); + } + + targetStack.forEach(({ func, line, args }, index) => { + if (index < this.frames.length) { + const frame = this.frames[index]; + if (frame.functionName !== func) { + frame.updateLine(line); + } + frame.setActive(index === targetStack.length - 1); + } else { + this.pushFrame(func, line, args); + } + }); + + if (this.frames.length > 0) { + this.frames[this.frames.length - 1].setActive(true); + } + } + } + + // ============================================================================ + // Sampling Panel Component + // ============================================================================ + + class DOMSamplingPanel { + constructor() { + this.samples = []; + this.functionCounts = {}; + this.totalSamples = 0; + this.sampleInterval = TIMINGS.sampleIntervalDefault; + this.groundTruthFunctions = new Set(); + this.bars = {}; + + this.element = document.createElement("div"); + this.element.className = "sampling-panel"; + + const header = document.createElement("div"); + header.className = "sampling-header"; + + const title = document.createElement("h3"); + title.className = "sampling-title"; + title.textContent = "Sampling Profiler"; + header.appendChild(title); + + const stats = document.createElement("div"); + stats.className = "sampling-stats"; + + this.sampleCountEl = document.createElement("span"); + this.sampleCountEl.textContent = "Samples: 0"; + stats.appendChild(this.sampleCountEl); + + this.intervalEl = document.createElement("span"); + this.intervalEl.textContent = `Interval: ${this.sampleInterval}ms`; + stats.appendChild(this.intervalEl); + + this.missedFunctionsEl = document.createElement("span"); + this.missedFunctionsEl.className = "missed"; + stats.appendChild(this.missedFunctionsEl); + + header.appendChild(stats); + this.element.appendChild(header); + + this.barsContainer = document.createElement("div"); + this.barsContainer.className = "sampling-bars"; + this.element.appendChild(this.barsContainer); + } + + setSampleInterval(interval) { + this.sampleInterval = interval; + this.intervalEl.textContent = `Interval: ${interval}ms`; + } + + setGroundTruth(allFunctions) { + this.groundTruthFunctions = new Set(allFunctions); + this._updateMissedCount(); + } + + addSample(stack) { + this.totalSamples++; + this.sampleCountEl.textContent = `Samples: ${this.totalSamples}`; + + stack.forEach((frame) => { + const funcName = frame.func; + this.functionCounts[funcName] = + (this.functionCounts[funcName] || 0) + 1; + }); + + this._updateBars(); + this._updateMissedCount(); + } + + reset() { + this.samples = []; + this.functionCounts = {}; + this.totalSamples = 0; + this.sampleCountEl.textContent = "Samples: 0"; + this.missedFunctionsEl.textContent = ""; + this.barsContainer.innerHTML = ""; + this.bars = {}; + } + + _updateMissedCount() { + if (this.groundTruthFunctions.size === 0) return; + + const capturedFunctions = new Set(Object.keys(this.functionCounts)); + const notYetSeen = [...this.groundTruthFunctions].filter( + (f) => !capturedFunctions.has(f), + ); + + if (notYetSeen.length > 0) { + this.missedFunctionsEl.textContent = `Not yet seen: ${notYetSeen.length}`; + this.missedFunctionsEl.classList.add("missed"); + this.missedFunctionsEl.style.color = ""; + } else if (this.totalSamples > 0) { + this.missedFunctionsEl.textContent = "All captured!"; + this.missedFunctionsEl.classList.remove("missed"); + this.missedFunctionsEl.style.color = "var(--color-green)"; + } else { + this.missedFunctionsEl.textContent = ""; + } + } + + _updateBars() { + const sorted = Object.entries(this.functionCounts).sort( + (a, b) => b[1] - a[1], + ); + + sorted.forEach(([funcName, count], index) => { + const percentage = + this.totalSamples > 0 ? count / this.totalSamples : 0; + + if (!this.bars[funcName]) { + const row = this._createBarRow(funcName); + this.barsContainer.appendChild(row); + this.bars[funcName] = row; + } + + const row = this.bars[funcName]; + const barFill = row.querySelector(".bar-fill"); + barFill.style.width = `${percentage * 100}%`; + + const percentEl = row.querySelector(".bar-percent"); + percentEl.textContent = `${(percentage * 100).toFixed(0)}%`; + + const currentIndex = Array.from(this.barsContainer.children).indexOf( + row, + ); + if (currentIndex !== index) { + this.barsContainer.insertBefore( + row, + this.barsContainer.children[index], + ); + } + }); + } + + _createBarRow(funcName) { + const row = document.createElement("div"); + row.className = "sampling-bar-row"; + row.dataset.function = funcName; + + const label = document.createElement("span"); + label.className = "bar-label"; + label.textContent = funcName; + row.appendChild(label); + + const barContainer = document.createElement("div"); + barContainer.className = "bar-container"; + + const barFill = document.createElement("div"); + barFill.className = "bar-fill"; + barFill.style.backgroundColor = getFunctionColor(funcName); + barContainer.appendChild(barFill); + + row.appendChild(barContainer); + + const percent = document.createElement("span"); + percent.className = "bar-percent"; + percent.textContent = "0%"; + row.appendChild(percent); + + return row; + } + + getTargetPosition() { + const rect = this.barsContainer.getBoundingClientRect(); + return { x: rect.left + rect.width / 2, y: rect.top + 50 }; + } + + showImpactEffect(position) { + const impact = document.createElement("div"); + impact.className = "impact-circle"; + impact.style.position = "fixed"; + impact.style.left = `${position.x}px`; + impact.style.top = `${position.y}px`; + + // Append to barsContainer parent to avoid triggering scroll + this.element.appendChild(impact); + + impact.animate( + [ + { transform: "translate(-50%, -50%) scale(1)", opacity: 0.6 }, + { transform: "translate(-50%, -50%) scale(4)", opacity: 0 }, + ], + { + duration: 300, + easing: "ease-out", + }, + ).onfinish = () => impact.remove(); + } + } + + // ============================================================================ + // Control Panel Component + // ============================================================================ + + class ControlPanel { + constructor( + container, + onPlay, + onPause, + onReset, + onSpeedChange, + onSeek, + onStep, + onSampleIntervalChange = null, + ) { + this.container = container; + this.onPlay = onPlay; + this.onPause = onPause; + this.onReset = onReset; + this.onSpeedChange = onSpeedChange; + this.onSeek = onSeek; + this.onStep = onStep; + this.onSampleIntervalChange = onSampleIntervalChange; + + this.isPlaying = false; + this.speed = TIMINGS.defaultSpeed; + + this._createControls(); + } + + _createControls() { + const panel = document.createElement("div"); + panel.id = "control-panel"; + + const sampleIntervalHtml = this.onSampleIntervalChange + ? ` +
+ + + ${TIMINGS.sampleIntervalDefault}ms +
+ ` + : ""; + + panel.innerHTML = ` +
+ + + +
+ + ${sampleIntervalHtml} + +
+ + 0ms +
+ `; + + this.container.appendChild(panel); + + this.playPauseBtn = panel.querySelector("#play-pause-btn"); + this.resetBtn = panel.querySelector("#reset-btn"); + this.stepBtn = panel.querySelector("#step-btn"); + this.scrubber = panel.querySelector("#timeline-scrubber"); + this.timeDisplay = panel.querySelector("#time-display"); + + this.playPauseBtn.addEventListener("click", () => + this._togglePlayPause(), + ); + this.resetBtn.addEventListener("click", () => this._handleReset()); + this.stepBtn.addEventListener("click", () => this._handleStep()); + this.scrubber.addEventListener("input", (e) => this._handleSeek(e)); + + if (this.onSampleIntervalChange) { + this.sampleIntervalSlider = panel.querySelector("#sample-interval"); + this.intervalDisplay = panel.querySelector("#interval-display"); + this.sampleIntervalSlider.addEventListener("input", (e) => + this._handleSampleIntervalChange(e), + ); + } + } + + _handleSampleIntervalChange(e) { + const interval = parseInt(e.target.value); + this.intervalDisplay.textContent = `${interval}ms`; + this.onSampleIntervalChange(interval); + } + + _togglePlayPause() { + this.isPlaying = !this.isPlaying; + + if (this.isPlaying) { + this.playPauseBtn.textContent = "⏸ Pause"; + this.playPauseBtn.classList.add("active"); + this.onPlay(); + } else { + this.playPauseBtn.textContent = "▶ Play"; + this.playPauseBtn.classList.remove("active"); + this.onPause(); + } + } + + _handleReset() { + this.isPlaying = false; + this.playPauseBtn.textContent = "▶ Play"; + this.playPauseBtn.classList.remove("active"); + this.scrubber.value = 0; + this.timeDisplay.textContent = "0ms"; + this.onReset(); + } + + _handleStep() { + if (this.onStep) this.onStep(); + } + + _handleSeek(e) { + const percentage = parseFloat(e.target.value); + this.onSeek(percentage / 100); + } + + updateTimeDisplay(currentTime, totalTime) { + this.timeDisplay.textContent = `${Math.floor(currentTime)}ms / ${Math.floor(totalTime)}ms`; + const percentage = (currentTime / totalTime) * 100; + this.scrubber.value = percentage; + } + + setDuration(duration) { + this.duration = duration; + } + + pause() { + if (this.isPlaying) this._togglePlayPause(); + } + + destroy() { + const panel = this.container.querySelector("#control-panel"); + if (panel) panel.remove(); + } + } + + // ============================================================================ + // Visual Effects Manager + // ============================================================================ + + class VisualEffectsManager { + constructor(container) { + this.container = container; + this.flyingAnimationInProgress = false; + + this.flashOverlay = document.createElement("div"); + this.flashOverlay.className = "flash-overlay"; + this.container.appendChild(this.flashOverlay); + } + + triggerSamplingEffect(stackViz, samplingPanel, currentTime, trace) { + if (this.flyingAnimationInProgress) return; + + const stack = trace.getStackAt(currentTime); + + if (stack.length === 0) { + samplingPanel.addSample(stack); + return; + } + + this.flyingAnimationInProgress = true; + stackViz.flashAll(); + + const clone = stackViz.createStackClone(this.container); + const targetPosition = samplingPanel.getTargetPosition(); + + this._animateFlash(); + this._animateFlyingStack(clone, targetPosition, () => { + samplingPanel.showImpactEffect(targetPosition); + clone.remove(); + + const currentStack = trace.getStackAt(currentTime); + samplingPanel.addSample(currentStack); + this.flyingAnimationInProgress = false; + }); + } + + _animateFlash() { + anim.to(this.flashOverlay, { opacity: 0.1 }, 0).onfinish = () => { + anim.to(this.flashOverlay, { opacity: 0 }, 150, "easeOutQuad"); + }; + } + + _animateFlyingStack(clone, targetPosition, onComplete) { + const containerRect = this.container.getBoundingClientRect(); + const cloneRect = clone.getBoundingClientRect(); + + // Convert viewport coordinates to container-relative + const startX = cloneRect.left - containerRect.left + cloneRect.width / 2; + const startY = cloneRect.top - containerRect.top + cloneRect.height / 2; + const targetX = targetPosition.x - containerRect.left; + const targetY = targetPosition.y - containerRect.top; + + const deltaX = targetX - startX; + const deltaY = targetY - startY; + + anim.to( + clone, + { + x: deltaX, + y: deltaY, + scale: 0.3, + opacity: 0.6, + }, + TIMINGS.sampleToFlame, + "easeOutCubic", + onComplete, + ); + } + } + + // ============================================================================ + // Main Visualization Class + // ============================================================================ + + class SamplingVisualization { + constructor(container) { + this.container = container; + + this.trace = new ExecutionTrace(DEMO_SIMPLE.source, DEMO_SIMPLE.trace); + + this.currentTime = 0; + this.isPlaying = false; + this.playbackSpeed = TIMINGS.defaultSpeed; + this.eventIndex = 0; + + this.sampleInterval = TIMINGS.sampleIntervalDefault; + this.lastSampleTime = 0; + + this._createLayout(); + + this.effectsManager = new VisualEffectsManager(this.vizColumn); + + this.lastTime = performance.now(); + this._animate(); + } + + _createLayout() { + this.codePanel = new CodePanel(this.trace.source); + this.container.appendChild(this.codePanel.element); + + this.vizColumn = document.createElement("div"); + this.vizColumn.className = "viz-column"; + this.container.appendChild(this.vizColumn); + + const stackSection = document.createElement("div"); + stackSection.className = "stack-section"; + + const stackTitle = document.createElement("div"); + stackTitle.className = "stack-section-title"; + stackTitle.textContent = "Call Stack"; + stackSection.appendChild(stackTitle); + + this.stackViz = new DOMStackVisualization(); + stackSection.appendChild(this.stackViz.element); + this.vizColumn.appendChild(stackSection); + + this.samplingPanel = new DOMSamplingPanel(); + this.samplingPanel.setGroundTruth(this._getGroundTruthFunctions()); + this.vizColumn.appendChild(this.samplingPanel.element); + + this.controls = new ControlPanel( + this.vizColumn, + () => this.play(), + () => this.pause(), + () => this.reset(), + (speed) => this.setSpeed(speed), + (progress) => this.seek(progress), + () => this.step(), + (interval) => this.setSampleInterval(interval), + ); + this.controls.setDuration(this.trace.duration); + } + + _getGroundTruthFunctions() { + const functions = new Set(); + this.trace.events.forEach((event) => { + if (event.type === "call") { + functions.add(event.functionName); + } + }); + return [...functions]; + } + + play() { + this.isPlaying = true; + } + + pause() { + this.isPlaying = false; + } + + reset() { + this.currentTime = 0; + this.eventIndex = 0; + this.isPlaying = false; + this.lastSampleTime = 0; + this.stackViz.clear(); + this.codePanel.reset(); + this.samplingPanel.reset(); + this.controls.updateTimeDisplay(0, this.trace.duration); + } + + setSpeed(speed) { + this.playbackSpeed = speed; + } + + setSampleInterval(interval) { + this.sampleInterval = interval; + this.samplingPanel.setSampleInterval(interval); + } + + seek(progress) { + this.currentTime = progress * this.trace.duration; + this.eventIndex = 0; + this.lastSampleTime = 0; + this._rebuildState(); + } + + step() { + this.pause(); + + const nextEvent = this.trace.getNextEvent(this.currentTime); + + if (nextEvent) { + // Calculate delta to reach next event + epsilon + const targetTime = nextEvent.timestamp + 0.1; + const delta = targetTime - this.currentTime; + if (delta > 0) { + this._advanceTime(delta); + } + } + } + + _animate(currentTime = performance.now()) { + const deltaTime = currentTime - this.lastTime; + this.lastTime = currentTime; + + this.update(deltaTime); + requestAnimationFrame((t) => this._animate(t)); + } + + update(deltaTime) { + if (!this.isPlaying) { + this.controls.updateTimeDisplay(this.currentTime, this.trace.duration); + return; + } + + const virtualDelta = deltaTime * this.playbackSpeed; + this._advanceTime(virtualDelta); + } + + _advanceTime(virtualDelta) { + this.currentTime += virtualDelta; + + if (this.currentTime >= this.trace.duration) { + this.currentTime = this.trace.duration; + this.isPlaying = false; + this.controls.pause(); + } + + while (this.eventIndex < this.trace.events.length) { + const event = this.trace.events[this.eventIndex]; + + if (event.timestamp > this.currentTime) break; + + this._processEvent(event); + this.eventIndex++; + } + + this.controls.updateTimeDisplay(this.currentTime, this.trace.duration); + + if (this.currentTime - this.lastSampleTime >= this.sampleInterval) { + this._takeSample(); + this.lastSampleTime = this.currentTime; + } + } + + _processEvent(event) { + this.stackViz.processEvent(event); + + if (event.type === "call") { + this.codePanel.highlightLine(event.lineno); + } else if (event.type === "return") { + const currentStack = this.trace.getStackAt(this.currentTime); + if (currentStack.length > 0) { + this.codePanel.highlightLine( + currentStack[currentStack.length - 1].line, + ); + } else { + this.codePanel.highlightLine(null); + } + } else if (event.type === "line") { + this.codePanel.highlightLine(event.lineno); + } + } + + _takeSample() { + this.effectsManager.triggerSamplingEffect( + this.stackViz, + this.samplingPanel, + this.currentTime, + this.trace, + ); + } + + _rebuildState() { + this.stackViz.clear(); + this.codePanel.reset(); + this.samplingPanel.reset(); + + for (let t = 0; t < this.currentTime; t += this.sampleInterval) { + const stack = this.trace.getStackAt(t); + this.samplingPanel.addSample(stack); + this.lastSampleTime = t; + } + + const stack = this.trace.getStackAt(this.currentTime); + this.stackViz.updateToMatch(stack); + + if (stack.length > 0) { + this.codePanel.highlightLine(stack[stack.length - 1].line); + } + + this.eventIndex = this.trace.getEventsUntil(this.currentTime).length; + } + } + + // ============================================================================ + // Initialize + // ============================================================================ + + function init() { + // If trace data hasn't been injected yet (local dev), don't initialize + if (!DEMO_SIMPLE) return; + + const appContainer = document.getElementById("sampling-profiler-viz"); + if (appContainer) { + new SamplingVisualization(appContainer); + } + } + + if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", init); + } else { + init(); + } +})(); diff --git a/Doc/_static/tachyon-example-flamegraph.html b/Doc/_static/tachyon-example-flamegraph.html new file mode 100644 index 000000000000000..701c959bda3ccc0 --- /dev/null +++ b/Doc/_static/tachyon-example-flamegraph.html @@ -0,0 +1,3260 @@ + + + + + + Tachyon Profiler - Flamegraph Report + + + + + + + +
+ +
+
+ + Tachyon + + Flamegraph Report +
+
+ + +
+
+ + + + + + + + +
+
+ + +
+ + + + +
+
+
+
+ + +
+ + Tachyon Profiler + + + Python Sampling Profiler + + + + + +
+
+ + + + diff --git a/Doc/_static/tachyon-example-heatmap.html b/Doc/_static/tachyon-example-heatmap.html new file mode 100644 index 000000000000000..f725e9475009fe7 --- /dev/null +++ b/Doc/_static/tachyon-example-heatmap.html @@ -0,0 +1,3804 @@ + + + + + + /tmp/tachyon_selfcontained.py - Heatmap + + + +
+ +
+
+ + Tachyon + + /tmp/tachyon_selfcontained.py +
+
+ + + + + + + + + + + +
+
+ + +
+
+
+
1,054
+
Self Samples
+
+
+
4,236
+
Cumulative
+
+
+
24
+
Lines Hit
+
+
+
100.00%
+
% of Total
+
+
+
206
+
Max Self
+
+
+
1054
+
Max Total
+
+
+
+ + +
+
+ Intensity: +
+
+ Cold + + Hot +
+ +
+
+ Self Time +
+ Total Time +
+
+ Show All +
+ Hot Only +
+
+ Heat +
+ Specialization +
+ + +
+
+
+ + +
+
+
Line
+
Self
+
Total
+
Code
+
+
+
1
+
+
+
+
"""
+ +
+
+
2
+
+
+
+
Tachyon Demo - Self-contained profiling example.
+ +
+
+
3
+
+
+
+
Pure Python with no external imports for clean heatmap output.
+ +
+
+
4
+
+
+
+
"""
+ +
+
+
5
+
+
+
+
+ +
+
+
6
+
+
+
+
+ +
+
+
7
+
1
+
1
+ +
def fibonacci(n):
+ +
+ +
+
8
+
+
+
+
"""Recursive fibonacci - creates deep call stacks."""
+ +
+
+
9
+
3
+
3
+ +
if n <= 1:
+ +
+ +
+
10
+
17
+
17
+ +
return n
+
+
+ +
+
11
+
206
+
227
+ +
return fibonacci(n - 1) + fibonacci(n - 2)
+
+
+ +
+
12
+
+
+
+
+ +
+
+
13
+
+
+
+
+ +
+
+
14
+
+
+
+
def bubble_sort(arr):
+ +
+
+
15
+
+
+
+
"""Classic bubble sort - O(n^2) comparison sorting."""
+ +
+
+
16
+
+
+
+
n = len(arr)
+ +
+
+
17
+
+
+
+
for i in range(n):
+ +
+
+
18
+
2
+
2
+ +
for j in range(0, n - i - 1):
+ +
+ +
+
19
+
8
+
8
+ +
if arr[j] > arr[j + 1]:
+
+
+ +
+
20
+
2
+
2
+ +
arr[j], arr[j + 1] = arr[j + 1], arr[j]
+ +
+ +
+
21
+
+
+
+
return arr
+ +
+
+
22
+
+
+
+
+ +
+
+
23
+
+
+
+
+ +
+
+
24
+
+
+
+
def matrix_multiply(a, b):
+ +
+
+
25
+
+
+
+
"""Matrix multiplication using nested loops."""
+ +
+
+
26
+
+
+
+
rows_a, cols_a = len(a), len(a[0])
+ +
+
+
27
+
+
+
+
rows_b, cols_b = len(b), len(b[0])
+ +
+
+
28
+
+
+
+
result = [[0] * cols_b for _ in range(rows_a)]
+ +
+
+
29
+
+
+
+
+ +
+
+
30
+
+
+
+
for i in range(rows_a):
+ +
+
+
31
+
+
+
+
for j in range(cols_b):
+ +
+
+
32
+
8
+
8
+ +
for k in range(cols_a):
+ +
+ +
+
33
+
62
+
62
+ +
result[i][j] += a[i][k] * b[k][j]
+
+
+ +
+
34
+
+
+
+
return result
+ +
+
+
35
+
+
+
+
+ +
+
+
36
+
+
+
+
+ +
+
+
37
+
+
+
+
def prime_sieve(limit):
+ +
+
+
38
+
+
+
+
"""Sieve of Eratosthenes - find all primes up to limit."""
+ +
+
+
39
+
+
+
+
is_prime = [True] * (limit + 1)
+ +
+
+
40
+
+
+
+
is_prime[0] = is_prime[1] = False
+ +
+
+
41
+
+
+
+
+ +
+
+
42
+
+
+
+
for num in range(2, int(limit ** 0.5) + 1):
+ +
+
+
43
+
+
+
+
if is_prime[num]:
+ +
+
+
44
+
1
+
1
+ +
for multiple in range(num * num, limit + 1, num):
+
+
+ +
+
45
+
+
+
+
is_prime[multiple] = False
+ +
+
+
46
+
+
+
+
+ +
+
+
47
+
2
+
2
+ +
return [num for num, prime in enumerate(is_prime) if prime]
+ +
+ +
+
48
+
+
+
+
+ +
+
+
49
+
+
+
+
+ +
+
+
50
+
+
+
+
def string_processing(iterations):
+ +
+
+
51
+
+
+
+
"""String operations - concatenation and formatting."""
+ +
+
+
52
+
+
+
+
for _ in range(iterations):
+ +
+
+
53
+
+
+
+
result = ""
+ +
+
+
54
+
+
+
+
for i in range(50):
+ +
+
+
55
+
91
+
91
+ +
result += f"item_{i}_"
+
+
+ +
+
56
+
22
+
22
+ +
parts = result.split("_")
+ +
+ +
+
57
+
10
+
10
+ +
joined = "-".join(parts)
+ +
+ +
+
58
+
+
+
+
+ +
+
+
59
+
+
+
+
+ +
+
+
60
+
+
+
+
def list_operations(iterations):
+ +
+
+
61
+
+
+
+
"""List comprehensions and operations."""
+ +
+
+
62
+
+
+
+
for _ in range(iterations):
+ +
+
+
63
+
118
+
118
+ +
squares = [x ** 2 for x in range(500)]
+ +
+ +
+
64
+
119
+
119
+ +
evens = [x for x in squares if x % 2 == 0]
+
+
+ +
+
65
+
12
+
12
+ +
total = sum(evens)
+ +
+ +
+
66
+
+
+
+
+ +
+
+
67
+
+
+
+
+ +
+
+
68
+
+
+
+
def dict_operations(iterations):
+ +
+
+
69
+
+
+
+
"""Dictionary creation and lookups."""
+ +
+
+
70
+
+
+
+
for _ in range(iterations):
+ +
+
+
71
+
206
+
206
+ +
data = {f"key_{i}": i ** 2 for i in range(200)}
+
+
+ +
+
72
+
158
+
158
+ +
values = [data[f"key_{i}"] for i in range(200)]
+ +
+ +
+
73
+
5
+
5
+ +
total = sum(values)
+ +
+ +
+
74
+
+
+
+
+ +
+
+
75
+
+
+
+
+ +
+
+
76
+
+
+
+
def compute_heavy():
+ +
+
+
77
+
+
+
+
"""CPU-intensive computation section."""
+ +
+
+
78
+
1
+
301
+ +
fibonacci(30)
+
+
+ +
+
79
+
+
+
+
+ +
+
+
80
+
+
+
+
size = 60
+ +
+
+
81
+
+
+
+
a = [[i + j for j in range(size)] for i in range(size)]
+ +
+
+
82
+
+
+
+
b = [[i * j for j in range(size)] for i in range(size)]
+ +
+
+
83
+
+
+
+
matrix_multiply(a, b)
+ +
+
+
84
+
+
+
+
+ +
+
+
85
+
+
+
+
prime_sieve(10000)
+ +
+
+
86
+
+
+
+
+ +
+
+
87
+
+
+
+
+ +
+
+
88
+
+
+
+
def data_processing():
+ +
+
+
89
+
+
+
+
"""Data structure operations."""
+ +
+
+
90
+
+
753
+ +
string_processing(2000)
+
+
+ +
+
91
+
+
+
+
list_operations(1500)
+ +
+
+
92
+
+
+
+
dict_operations(1000)
+ +
+
+
93
+
+
+
+
+ +
+
+
94
+
+
+
+
data = list(range(800))
+ +
+
+
95
+
+
+
+
for _ in range(3):
+ +
+
+
96
+
+
+
+
data = data[::-1]
+ +
+
+
97
+
+
+
+
bubble_sort(data[:200])
+ +
+
+
98
+
+
+
+
+ +
+
+
99
+
+
+
+
+ +
+
+
100
+
+
+
+
def main():
+ +
+
+
101
+
+
+
+
"""Main entry point."""
+ +
+
+
102
+
+
1,054
+ +
compute_heavy()
+
+
+ +
+
103
+
+
+
+
data_processing()
+ +
+
+
104
+
+
+
+
+ +
+
+
105
+
+
+
+
+ +
+
+
106
+
+
+
+
if __name__ == "__main__":
+ +
+
+
107
+
+
1,054
+ +
main()
+
+
+ + +
+
+ + + + diff --git a/Doc/about.rst b/Doc/about.rst index 8f635d7f743a982..5c1b497ca6bcea8 100644 --- a/Doc/about.rst +++ b/Doc/about.rst @@ -32,8 +32,9 @@ Contributors to the Python documentation ---------------------------------------- Many people have contributed to the Python language, the Python standard -library, and the Python documentation. See :source:`Misc/ACKS` in the Python -source distribution for a partial list of contributors. +library, and the Python documentation. See the `CPython +GitHub repository `__ +for a partial list of contributors. It is only with the input and contributions of the Python community that Python has such wonderful documentation -- Thank You! diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 5d0f68ca69675e5..a6ea0a72e76f9db 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -9,9 +9,12 @@ stability. In order to maintain this reputation, the developers would like to know of any deficiencies you find in Python. It can be sometimes faster to fix bugs yourself and contribute patches to -Python as it streamlines the process and involves less people. Learn how to +Python as it streamlines the process and involves fewer people. Learn how to :ref:`contribute `. + +.. _reporting-documentation-bugs: + Documentation bugs ================== @@ -19,6 +22,12 @@ If you find a bug in this documentation or would like to propose an improvement, please submit a bug report on the :ref:`issue tracker `. If you have a suggestion on how to fix it, include that as well. +.. only:: translation + + If the bug or suggested improvement concerns the translation of this + documentation, submit the report to the + `translation’s repository `_ instead. + You can also open a discussion item on our `Documentation Discourse forum `_. @@ -37,8 +46,8 @@ tracker `_. `Helping with Documentation `_ Comprehensive guide for individuals that are interested in contributing to Python documentation. - `Documentation Translations `_ - A list of GitHub pages for documentation translation and their primary contacts. + `Documentation Translations `_ + A list of GitHub pages for documentation translation and their coordination teams. .. _using-the-tracker: diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 59d913a04623829..09c9ed3ca54cf69 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -2,7 +2,7 @@ .. _allocating-objects: -Allocating Objects on the Heap +Allocating objects on the heap ============================== @@ -140,10 +140,6 @@ Allocating Objects on the Heap * :c:member:`~PyTypeObject.tp_alloc` -.. c:function:: void PyObject_Del(void *op) - - Same as :c:func:`PyObject_Free`. - .. c:var:: PyObject _Py_NoneStruct Object which is visible in Python as ``None``. This should only be accessed @@ -156,3 +152,37 @@ Allocating Objects on the Heap :ref:`moduleobjects` To allocate and create extension modules. + +Soft-deprecated aliases +^^^^^^^^^^^^^^^^^^^^^^^ + +.. soft-deprecated:: 3.15 + +These are aliases to existing functions and macros. +They exist solely for backwards compatibility. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * Soft-deprecated alias + * Function + * * .. c:macro:: PyObject_NEW(type, typeobj) + * :c:macro:`PyObject_New` + * * .. c:macro:: PyObject_NEW_VAR(type, typeobj, n) + * :c:macro:`PyObject_NewVar` + * * .. c:macro:: PyObject_INIT(op, typeobj) + * :c:func:`PyObject_Init` + * * .. c:macro:: PyObject_INIT_VAR(op, typeobj, n) + * :c:func:`PyObject_InitVar` + * * .. c:macro:: PyObject_MALLOC(n) + * :c:func:`PyObject_Malloc` + * * .. c:macro:: PyObject_REALLOC(p, n) + * :c:func:`PyObject_Realloc` + * * .. c:macro:: PyObject_FREE(p) + * :c:func:`PyObject_Free` + * * .. c:macro:: PyObject_DEL(p) + * :c:func:`PyObject_Free` + * * .. c:macro:: PyObject_Del(p) + * :c:func:`PyObject_Free` diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst index 96050f59bd52508..cb98d4307ee5224 100644 --- a/Doc/c-api/apiabiversion.rst +++ b/Doc/c-api/apiabiversion.rst @@ -34,6 +34,23 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. This can be ``0xA`` for alpha, ``0xB`` for beta, ``0xC`` for release candidate or ``0xF`` for final. + + .. c:namespace:: NULL + .. c:macro:: PY_RELEASE_LEVEL_ALPHA + :no-typesetting: + .. c:macro:: PY_RELEASE_LEVEL_BETA + :no-typesetting: + .. c:macro:: PY_RELEASE_LEVEL_GAMMA + :no-typesetting: + .. c:macro:: PY_RELEASE_LEVEL_FINAL + :no-typesetting: + + For completeness, the values are available as macros: + :c:macro:`!PY_RELEASE_LEVEL_ALPHA` (``0xA``), + :c:macro:`!PY_RELEASE_LEVEL_BETA` (``0xB``), + :c:macro:`!PY_RELEASE_LEVEL_GAMMA` (``0xC``), and + :c:macro:`!PY_RELEASE_LEVEL_FINAL` (``0xF``). + .. c:macro:: PY_RELEASE_SERIAL The ``2`` in ``3.4.1a2``. Zero for final releases. @@ -46,6 +63,12 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. Use this for numeric comparisons, for example, ``#if PY_VERSION_HEX >= ...``. +.. c:macro:: PY_VERSION + + The Python version as a string, for example, ``"3.4.1a2"``. + +These macros are defined in :source:`Include/patchlevel.h`. + Run-time version ---------------- diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 3429a4eb652709b..58456a36b96c151 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -160,7 +160,7 @@ There are three ways strings and buffers can be converted to C: ``w*`` (read-write :term:`bytes-like object`) [Py_buffer] This format accepts any object which implements the read-write buffer interface. It fills a :c:type:`Py_buffer` structure provided by the caller. - The buffer may contain embedded null bytes. The caller have to call + The buffer may contain embedded null bytes. The caller has to call :c:func:`PyBuffer_Release` when it is done with the buffer. ``es`` (:class:`str`) [const char \*encoding, char \*\*buffer] @@ -305,7 +305,7 @@ the minimal value for the corresponding signed integer type of the same size. ``D`` (:class:`complex`) [Py_complex] Convert a Python complex number to a C :c:type:`Py_complex` structure. -.. deprecated:: next +.. deprecated:: 3.15 For unsigned integer formats ``B``, ``H``, ``I``, ``k`` and ``K``, :exc:`DeprecationWarning` is emitted when the value is larger than @@ -516,6 +516,28 @@ API Functions } +.. c:function:: int PyArg_ParseArray(PyObject *const *args, Py_ssize_t nargs, const char *format, ...) + + Parse the parameters of a function that takes only array parameters into + local variables (that is, a function using the :c:macro:`METH_FASTCALL` + calling convention). + Returns true on success; on failure, it returns false and raises the + appropriate exception. + + .. versionadded:: 3.15 + + +.. c:function:: int PyArg_ParseArrayAndKeywords(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, const char *format, const char * const *kwlist, ...) + + Parse the parameters of a function that takes both array and keyword + parameters into local variables (that is, a function using the + :c:macro:`METH_FASTCALL` ``|`` :c:macro:`METH_KEYWORDS` calling convention). + Returns true on success; on failure, it returns false and raises the + appropriate exception. + + .. versionadded:: 3.15 + + .. c:function:: int PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t max, ...) A simpler form of parameter retrieval which does not use a format string to diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index d3081894eadaf5e..dc3e0f37c36c5b2 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -10,11 +10,6 @@ Buffer Protocol --------------- -.. sectionauthor:: Greg Stein -.. sectionauthor:: Benjamin Peterson -.. sectionauthor:: Stefan Krah - - Certain objects available in Python wrap access to an underlying memory array or *buffer*. Such objects include the built-in :class:`bytes` and :class:`bytearray`, and some extension types like :class:`array.array`. @@ -261,6 +256,12 @@ readonly, format MUST be consistent for all consumers. For example, :c:expr:`PyBUF_SIMPLE | PyBUF_WRITABLE` can be used to request a simple writable buffer. + .. c:macro:: PyBUF_WRITEABLE + + This is an alias to :c:macro:`PyBUF_WRITABLE`. + + .. soft-deprecated:: 3.13 + .. c:macro:: PyBUF_FORMAT Controls the :c:member:`~Py_buffer.format` field. If set, this field MUST @@ -501,10 +502,11 @@ Buffer-related functions *indices* must point to an array of ``view->ndim`` indices. -.. c:function:: int PyBuffer_FromContiguous(const Py_buffer *view, const void *buf, Py_ssize_t len, char fort) +.. c:function:: int PyBuffer_FromContiguous(const Py_buffer *view, const void *buf, Py_ssize_t len, char order) Copy contiguous *len* bytes from *buf* to *view*. - *fort* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering). + *order* can be ``'C'`` or ``'F'`` or ``'A'`` (for C-style or Fortran-style + ordering or either one). ``0`` is returned on success, ``-1`` on error. diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index e2b22ec3c794ae4..2b36da997d42956 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -44,6 +44,10 @@ Direct API functions On failure, return ``NULL`` with an exception set. + .. note:: + If the object implements the buffer protocol, then the buffer + must not be mutated while the bytearray object is being created. + .. c:function:: PyObject* PyByteArray_FromStringAndSize(const char *string, Py_ssize_t len) @@ -58,6 +62,10 @@ Direct API functions On failure, return ``NULL`` with an exception set. + .. note:: + If the object implements the buffer protocol, then the buffer + must not be mutated while the bytearray object is being created. + .. c:function:: Py_ssize_t PyByteArray_Size(PyObject *bytearray) @@ -70,6 +78,9 @@ Direct API functions ``NULL`` pointer. The returned array always has an extra null byte appended. + .. note:: + It is not thread-safe to mutate the bytearray object while using the returned char array. + .. c:function:: int PyByteArray_Resize(PyObject *bytearray, Py_ssize_t len) @@ -89,6 +100,9 @@ These macros trade safety for speed and they don't check pointers. Similar to :c:func:`PyByteArray_AsString`, but without error checking. + .. note:: + It is not thread-safe to mutate the bytearray object while using the returned char array. + .. c:function:: Py_ssize_t PyByteArray_GET_SIZE(PyObject *bytearray) diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index d47beee68eaa338..fa77d3d38ff89fd 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -47,6 +47,10 @@ called with a non-bytes parameter. *len* on success, and ``NULL`` on failure. If *v* is ``NULL``, the contents of the bytes object are uninitialized. + .. soft-deprecated:: 3.15 + Use the :c:type:`PyBytesWriter` API instead of + ``PyBytes_FromStringAndSize(NULL, len)``. + .. c:function:: PyObject* PyBytes_FromFormat(const char *format, ...) @@ -123,6 +127,10 @@ called with a non-bytes parameter. Return the bytes representation of object *o* that implements the buffer protocol. + .. note:: + If the object implements the buffer protocol, then the buffer + must not be mutated while the bytes object is being created. + .. c:function:: Py_ssize_t PyBytes_Size(PyObject *o) @@ -181,6 +189,9 @@ called with a non-bytes parameter. created, the old reference to *bytes* will still be discarded and the value of *\*bytes* will be set to ``NULL``; the appropriate exception will be set. + .. note:: + If *newpart* implements the buffer protocol, then the buffer + must not be mutated while the new bytes object is being created. .. c:function:: void PyBytes_ConcatAndDel(PyObject **bytes, PyObject *newpart) @@ -188,6 +199,10 @@ called with a non-bytes parameter. appended to *bytes*. This version releases the :term:`strong reference` to *newpart* (i.e. decrements its reference count). + .. note:: + If *newpart* implements the buffer protocol, then the buffer + must not be mutated while the new bytes object is being created. + .. c:function:: PyObject* PyBytes_Join(PyObject *sep, PyObject *iterable) @@ -206,6 +221,9 @@ called with a non-bytes parameter. .. versionadded:: 3.14 + .. note:: + If *iterable* objects implement the buffer protocol, then the buffers + must not be mutated while the new bytes object is being created. .. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize) @@ -219,3 +237,216 @@ called with a non-bytes parameter. reallocation fails, the original bytes object at *\*bytes* is deallocated, *\*bytes* is set to ``NULL``, :exc:`MemoryError` is set, and ``-1`` is returned. + + .. soft-deprecated:: 3.15 + Use the :c:type:`PyBytesWriter` API instead. + + +.. c:function:: PyObject *PyBytes_Repr(PyObject *bytes, int smartquotes) + + Get the string representation of *bytes*. This function is currently used to + implement :meth:`!bytes.__repr__` in Python. + + This function does not do type checking; it is undefined behavior to pass + *bytes* as a non-bytes object or ``NULL``. + + If *smartquotes* is true, the representation will use a double-quoted string + instead of single-quoted string when single-quotes are present in *bytes*. + For example, the byte string ``'Python'`` would be represented as + ``b"'Python'"`` when *smartquotes* is true, or ``b'\'Python\''`` when it is + false. + + On success, this function returns a :term:`strong reference` to a + :class:`str` object containing the representation. On failure, this + returns ``NULL`` with an exception set. + + +.. c:function:: PyObject *PyBytes_DecodeEscape(const char *s, Py_ssize_t len, const char *errors, Py_ssize_t unicode, const char *recode_encoding) + + Unescape a backslash-escaped string *s*. *s* must not be ``NULL``. + *len* must be the size of *s*. + + *errors* must be one of ``"strict"``, ``"replace"``, or ``"ignore"``. If + *errors* is ``NULL``, then ``"strict"`` is used by default. + + On success, this function returns a :term:`strong reference` to a Python + :class:`bytes` object containing the unescaped string. On failure, this + function returns ``NULL`` with an exception set. + + .. versionchanged:: 3.9 + *unicode* and *recode_encoding* are now unused. + + +.. _pybyteswriter: + +PyBytesWriter +------------- + +The :c:type:`PyBytesWriter` API can be used to create a Python :class:`bytes` +object. + +.. versionadded:: 3.15 + +.. c:type:: PyBytesWriter + + A bytes writer instance. + + The API is **not thread safe**: a writer should only be used by a single + thread at the same time. + + The instance must be destroyed by :c:func:`PyBytesWriter_Finish` on + success, or :c:func:`PyBytesWriter_Discard` on error. + + +Create, Finish, Discard +^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: PyBytesWriter* PyBytesWriter_Create(Py_ssize_t size) + + Create a :c:type:`PyBytesWriter` to write *size* bytes. + + If *size* is greater than zero, allocate *size* bytes, and set the + writer size to *size*. The caller is responsible to write *size* + bytes using :c:func:`PyBytesWriter_GetData`. + This function does not overallocate. + + On error, set an exception and return ``NULL``. + + *size* must be positive or zero. + +.. c:function:: PyObject* PyBytesWriter_Finish(PyBytesWriter *writer) + + Finish a :c:type:`PyBytesWriter` created by + :c:func:`PyBytesWriter_Create`. + + On success, return a Python :class:`bytes` object. + On error, set an exception and return ``NULL``. + + The writer instance is invalid after the call in any case. + No API can be called on the writer after :c:func:`PyBytesWriter_Finish`. + +.. c:function:: PyObject* PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) + + Similar to :c:func:`PyBytesWriter_Finish`, but resize the writer + to *size* bytes before creating the :class:`bytes` object. + +.. c:function:: PyObject* PyBytesWriter_FinishWithPointer(PyBytesWriter *writer, void *buf) + + Similar to :c:func:`PyBytesWriter_Finish`, but resize the writer + using *buf* pointer before creating the :class:`bytes` object. + + Set an exception and return ``NULL`` if *buf* pointer is outside the + internal buffer bounds. + + Function pseudo-code:: + + Py_ssize_t size = (char*)buf - (char*)PyBytesWriter_GetData(writer); + return PyBytesWriter_FinishWithSize(writer, size); + +.. c:function:: void PyBytesWriter_Discard(PyBytesWriter *writer) + + Discard a :c:type:`PyBytesWriter` created by :c:func:`PyBytesWriter_Create`. + + Do nothing if *writer* is ``NULL``. + + The writer instance is invalid after the call. + No API can be called on the writer after :c:func:`PyBytesWriter_Discard`. + +High-level API +^^^^^^^^^^^^^^ + +.. c:function:: int PyBytesWriter_WriteBytes(PyBytesWriter *writer, const void *bytes, Py_ssize_t size) + + Grow the *writer* internal buffer by *size* bytes, + write *size* bytes of *bytes* at the *writer* end, + and add *size* to the *writer* size. + + If *size* is equal to ``-1``, call ``strlen(bytes)`` to get the + string length. + + On success, return ``0``. + On error, set an exception and return ``-1``. + +.. c:function:: int PyBytesWriter_Format(PyBytesWriter *writer, const char *format, ...) + + Similar to :c:func:`PyBytes_FromFormat`, but write the output directly at + the writer end. Grow the writer internal buffer on demand. Then add the + written size to the writer size. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + +Getters +^^^^^^^ + +.. c:function:: Py_ssize_t PyBytesWriter_GetSize(PyBytesWriter *writer) + + Get the writer size. + + The function does not invalidate pointers returned by + :c:func:`PyBytesWriter_GetData`. + + The function cannot fail. + +.. c:function:: void* PyBytesWriter_GetData(PyBytesWriter *writer) + + Get the writer data: start of the internal buffer. + + The pointer remains valid until a :c:type:`PyBytesWriter` function other + than :c:func:`PyBytesWriter_GetData` or :c:func:`PyBytesWriter_GetSize` is + called on *writer*. + + The function cannot fail. + + +Low-level API +^^^^^^^^^^^^^ + +.. c:function:: int PyBytesWriter_Resize(PyBytesWriter *writer, Py_ssize_t size) + + Resize the writer to *size* bytes. It can be used to enlarge or to + shrink the writer. + This function typically overallocates to achieve amortized performance when + resizing multiple times. + + Newly allocated bytes are left uninitialized. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + *size* must be positive or zero. + +.. c:function:: int PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t grow) + + Resize the writer by adding *grow* bytes to the current writer size. + This function typically overallocates to achieve amortized performance when + resizing multiple times. + + Newly allocated bytes are left uninitialized. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + *size* can be negative to shrink the writer. + +.. c:function:: void* PyBytesWriter_GrowAndUpdatePointer(PyBytesWriter *writer, Py_ssize_t size, void *buf) + + Similar to :c:func:`PyBytesWriter_Grow`, but update also the *buf* + pointer. + + The *buf* pointer is moved if the internal buffer is moved in memory. + The *buf* relative position within the internal buffer is left + unchanged. + + On error, set an exception and return ``NULL``. + + *buf* must not be ``NULL``. + + Function pseudo-code:: + + Py_ssize_t pos = (char*)buf - (char*)PyBytesWriter_GetData(writer); + if (PyBytesWriter_Grow(writer, size) < 0) { + return NULL; + } + return (char*)PyBytesWriter_GetData(writer) + pos; diff --git a/Doc/c-api/call.rst b/Doc/c-api/call.rst index 7198d6bc056eb43..9838879a528934a 100644 --- a/Doc/c-api/call.rst +++ b/Doc/c-api/call.rst @@ -347,6 +347,8 @@ please see individual documentation for details. .. versionadded:: 3.9 +.. c:function:: PyObject* _PyObject_Vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) + :no-typesetting: .. c:function:: PyObject* PyObject_Vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) @@ -358,7 +360,12 @@ please see individual documentation for details. Return the result of the call on success, or raise an exception and return *NULL* on failure. - .. versionadded:: 3.9 + .. versionadded:: 3.8 as ``_PyObject_Vectorcall`` + + .. versionchanged:: 3.9 + + Renamed to the current name, without the leading underscore. + The old provisional name is :term:`soft deprecated`. .. c:function:: PyObject* PyObject_VectorcallDict(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwdict) diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst index 64dc4f5275b512c..03a848d68ed7aba 100644 --- a/Doc/c-api/capsule.rst +++ b/Doc/c-api/capsule.rst @@ -15,13 +15,19 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:type:: PyCapsule This subtype of :c:type:`PyObject` represents an opaque value, useful for C - extension modules who need to pass an opaque value (as a :c:expr:`void*` + extension modules which need to pass an opaque value (as a :c:expr:`void*` pointer) through Python code to other C code. It is often used to make a C function pointer defined in one module available to other modules, so the regular import mechanism can be used to access C APIs defined in dynamically loaded modules. +.. c:var:: PyTypeObject PyCapsule_Type + + The type object corresponding to capsule objects. This is the same object + as :class:`types.CapsuleType` in the Python layer. + + .. c:type:: PyCapsule_Destructor The type of a destructor callback for a capsule. Defined as:: diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index 61eb994c3709460..2501ed9580df898 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -7,7 +7,7 @@ Cell Objects "Cell" objects are used to implement variables referenced by multiple scopes. For each such variable, a cell object is created to store the value; the local -variables of each stack frame that references the value contains a reference to +variables of each stack frame that references the value contain a reference to the cells from outer scopes which also use that variable. When the value is accessed, the value contained in the cell is used instead of the cell object itself. This de-referencing of the cell object requires support from the diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 717b0da8f87c7f7..57b77f92a7d2e6a 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -7,8 +7,6 @@ Code Objects ------------ -.. sectionauthor:: Jeffrey Yasskin - Code objects are a low-level detail of the CPython implementation. Each one represents a chunk of executable code that hasn't yet been bound into a function. @@ -69,13 +67,14 @@ bound into a function. The old name is deprecated, but will remain available until the signature changes again. +.. c:function:: PyCodeObject* PyCode_NewWithPosOnlyArgs(...) + :no-typesetting: + .. c:function:: PyCodeObject* PyUnstable_Code_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable) Similar to :c:func:`PyUnstable_Code_New`, but with an extra "posonlyargcount" for positional-only arguments. The same caveats that apply to ``PyUnstable_Code_New`` also apply to this function. - .. index:: single: PyCode_NewWithPosOnlyArgs (C function) - .. versionadded:: 3.8 as ``PyCode_NewWithPosOnlyArgs`` .. versionchanged:: 3.11 @@ -211,6 +210,19 @@ bound into a function. .. versionadded:: 3.12 +.. c:function:: PyObject *PyCode_Optimize(PyObject *code, PyObject *consts, PyObject *names, PyObject *lnotab_obj) + + This is a function that does nothing. + + Prior to Python 3.10, this function would perform basic optimizations to a + code object. + + .. versionchanged:: 3.10 + This function now does nothing. + + .. soft-deprecated:: 3.13 + + .. _c_codeobject_flags: Code Object Flags @@ -287,9 +299,12 @@ These functions are part of the unstable C API tier: this functionality is a CPython implementation detail, and the API may change without deprecation warnings. +.. c:function:: Py_ssize_t _PyEval_RequestCodeExtraIndex(freefunc free) + :no-typesetting: + .. c:function:: Py_ssize_t PyUnstable_Eval_RequestCodeExtraIndex(freefunc free) - Return a new an opaque index value used to adding data to code objects. + Return a new opaque index value used to adding data to code objects. You generally call this function once (per interpreter) and use the result with ``PyCode_GetExtra`` and ``PyCode_SetExtra`` to manipulate @@ -299,8 +314,6 @@ may change without deprecation warnings. *free* will be called on non-``NULL`` data stored under the new index. Use :c:func:`Py_DecRef` when storing :c:type:`PyObject`. - .. index:: single: _PyEval_RequestCodeExtraIndex (C function) - .. versionadded:: 3.6 as ``_PyEval_RequestCodeExtraIndex`` .. versionchanged:: 3.12 @@ -309,6 +322,9 @@ may change without deprecation warnings. The old private name is deprecated, but will be available until the API changes. +.. c:function:: int _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) + :no-typesetting: + .. c:function:: int PyUnstable_Code_GetExtra(PyObject *code, Py_ssize_t index, void **extra) Set *extra* to the extra data stored under the given index. @@ -317,8 +333,6 @@ may change without deprecation warnings. If no data was set under the index, set *extra* to ``NULL`` and return 0 without setting an exception. - .. index:: single: _PyCode_GetExtra (C function) - .. versionadded:: 3.6 as ``_PyCode_GetExtra`` .. versionchanged:: 3.12 @@ -327,13 +341,14 @@ may change without deprecation warnings. The old private name is deprecated, but will be available until the API changes. +.. c:function:: int _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) + :no-typesetting: + .. c:function:: int PyUnstable_Code_SetExtra(PyObject *code, Py_ssize_t index, void *extra) Set the extra data stored under the given index to *extra*. Return 0 on success. Set an exception and return -1 on failure. - .. index:: single: _PyCode_SetExtra (C function) - .. versionadded:: 3.6 as ``_PyCode_SetExtra`` .. versionchanged:: 3.12 diff --git a/Doc/c-api/codec.rst b/Doc/c-api/codec.rst index 8ae5c4fecd6248f..35ee048bd5fa9f5 100644 --- a/Doc/c-api/codec.rst +++ b/Doc/c-api/codec.rst @@ -7,7 +7,7 @@ Codec registry and support functions Register a new codec search function. - As side effect, this tries to load the :mod:`!encodings` package, if not yet + As a side effect, this tries to load the :mod:`!encodings` package, if not yet done, to make sure that it is always first in the list of search functions. .. c:function:: int PyCodec_Unregister(PyObject *search_function) @@ -39,7 +39,7 @@ Codec registry and support functions *object* is passed through the decoder function found for the given *encoding* using the error handling method defined by *errors*. *errors* may be ``NULL`` to use the default method defined for the codec. Raises a - :exc:`LookupError` if no encoder can be found. + :exc:`LookupError` if no decoder can be found. Codec lookup API @@ -129,3 +129,13 @@ Registry API for Unicode encoding error handlers Replace the unicode encode error with ``\N{...}`` escapes. .. versionadded:: 3.5 + + +Codec utility variables +----------------------- + +.. c:var:: const char *Py_hexdigits + + A string constant containing the lowercase hexadecimal digits: ``"0123456789abcdef"``. + + .. versionadded:: 3.3 diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 16bd79475dc1e66..10f96c7cb75e882 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -3,92 +3,24 @@ .. _complexobjects: Complex Number Objects ----------------------- +====================== .. index:: pair: object; complex number -Python's complex number objects are implemented as two distinct types when -viewed from the C API: one is the Python object exposed to Python programs, and -the other is a C structure which represents the actual complex number value. -The API provides functions for working with both. - - -Complex Numbers as C Structures -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Note that the functions which accept these structures as parameters and return -them as results do so *by value* rather than dereferencing them through -pointers. This is consistent throughout the API. - - -.. c:type:: Py_complex - - The C structure which corresponds to the value portion of a Python complex - number object. Most of the functions for dealing with complex number objects - use structures of this type as input or output values, as appropriate. - - .. c:member:: double real - double imag - - The structure is defined as:: - - typedef struct { - double real; - double imag; - } Py_complex; - - -.. c:function:: Py_complex _Py_c_sum(Py_complex left, Py_complex right) - - Return the sum of two complex numbers, using the C :c:type:`Py_complex` - representation. - - -.. c:function:: Py_complex _Py_c_diff(Py_complex left, Py_complex right) - - Return the difference between two complex numbers, using the C - :c:type:`Py_complex` representation. - - -.. c:function:: Py_complex _Py_c_neg(Py_complex num) - - Return the negation of the complex number *num*, using the C - :c:type:`Py_complex` representation. - - -.. c:function:: Py_complex _Py_c_prod(Py_complex left, Py_complex right) - - Return the product of two complex numbers, using the C :c:type:`Py_complex` - representation. - - -.. c:function:: Py_complex _Py_c_quot(Py_complex dividend, Py_complex divisor) - - Return the quotient of two complex numbers, using the C :c:type:`Py_complex` - representation. - - If *divisor* is null, this method returns zero and sets - :c:data:`errno` to :c:macro:`!EDOM`. - - -.. c:function:: Py_complex _Py_c_pow(Py_complex num, Py_complex exp) - - Return the exponentiation of *num* by *exp*, using the C :c:type:`Py_complex` - representation. - - If *num* is null and *exp* is not a positive real number, - this method returns zero and sets :c:data:`errno` to :c:macro:`!EDOM`. - - Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows. +.. c:type:: PyComplexObject -Complex Numbers as Python Objects -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + This subtype of :c:type:`PyObject` represents a Python complex number object. + .. c:member:: Py_complex cval -.. c:type:: PyComplexObject + The complex number value, using the C :c:type:`Py_complex` representation. - This subtype of :c:type:`PyObject` represents a Python complex number object. + .. deprecated-removed:: 3.15 3.20 + Use :c:func:`PyComplex_AsCComplex` and + :c:func:`PyComplex_FromCComplex` to convert a + Python complex number to/from the C :c:type:`Py_complex` + representation. .. c:var:: PyTypeObject PyComplex_Type @@ -109,12 +41,6 @@ Complex Numbers as Python Objects :c:type:`PyComplexObject`. This function always succeeds. -.. c:function:: PyObject* PyComplex_FromCComplex(Py_complex v) - - Create a new Python complex number object from a C :c:type:`Py_complex` value. - Return ``NULL`` with an exception set on error. - - .. c:function:: PyObject* PyComplex_FromDoubles(double real, double imag) Return a new :c:type:`PyComplexObject` object from *real* and *imag*. @@ -153,6 +79,29 @@ Complex Numbers as Python Objects .. versionchanged:: 3.13 Use :meth:`~object.__complex__` if available. + +.. c:type:: Py_complex + + This C structure defines an export format for a Python complex + number object. + + .. c:member:: double real + double imag + + The structure is defined as:: + + typedef struct { + double real; + double imag; + } Py_complex; + + +.. c:function:: PyObject* PyComplex_FromCComplex(Py_complex v) + + Create a new Python complex number object from a C :c:type:`Py_complex` value. + Return ``NULL`` with an exception set on error. + + .. c:function:: Py_complex PyComplex_AsCComplex(PyObject *op) Return the :c:type:`Py_complex` value of the complex number *op*. @@ -169,3 +118,82 @@ Complex Numbers as Python Objects .. versionchanged:: 3.8 Use :meth:`~object.__index__` if available. + + +Complex Numbers as C Structures +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The API also provides functions for working with complex numbers, using the +:c:type:`Py_complex` representation. Note that the functions which accept +these structures as parameters and return them as results do so *by value* +rather than dereferencing them through pointers. + +Please note, that these functions are :term:`soft deprecated` since Python +3.15. Avoid using this API in a new code to do complex arithmetic: either use +the :ref:`Number Protocol ` API or use native complex types, like +:c:expr:`double complex`. + + +.. c:function:: Py_complex _Py_c_sum(Py_complex left, Py_complex right) + + Return the sum of two complex numbers, using the C :c:type:`Py_complex` + representation. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_diff(Py_complex left, Py_complex right) + + Return the difference between two complex numbers, using the C + :c:type:`Py_complex` representation. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_neg(Py_complex num) + + Return the negation of the complex number *num*, using the C + :c:type:`Py_complex` representation. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_prod(Py_complex left, Py_complex right) + + Return the product of two complex numbers, using the C :c:type:`Py_complex` + representation. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_quot(Py_complex dividend, Py_complex divisor) + + Return the quotient of two complex numbers, using the C :c:type:`Py_complex` + representation. + + If *divisor* is null, this method returns zero and sets + :c:data:`errno` to :c:macro:`!EDOM`. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_pow(Py_complex num, Py_complex exp) + + Return the exponentiation of *num* by *exp*, using the C :c:type:`Py_complex` + representation. + + If *num* is null and *exp* is not a positive real number, + this method returns zero and sets :c:data:`errno` to :c:macro:`!EDOM`. + + Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows. + + .. deprecated:: 3.15 + + +.. c:function:: double _Py_c_abs(Py_complex num) + + Return the absolute value of the complex number *num*. + + Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows. + + .. deprecated:: 3.15 diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index 880f7b15ce68e82..3f38411a52de6b0 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -109,11 +109,21 @@ Other Objects descriptor.rst slice.rst memoryview.rst + picklebuffer.rst weakref.rst capsule.rst + sentinel.rst frame.rst gen.rst coro.rst contextvars.rst - datetime.rst typehints.rst + + +C API for extension modules +=========================== + +.. toctree:: + + curses.rst + datetime.rst diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index c92ef4c653a675f..f7c8ef8b22b9556 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -41,7 +41,7 @@ The return value (*rv*) for these functions should be interpreted as follows: ``rv + 1`` bytes would have been needed to succeed. ``str[size-1]`` is ``'\0'`` in this case. -* When ``rv < 0``, "something bad happened." ``str[size-1]`` is ``'\0'`` in +* When ``rv < 0``, the output conversion failed and ``str[size-1]`` is ``'\0'`` in this case too, but the rest of *str* is undefined. The exact cause of the error depends on the underlying platform. @@ -105,7 +105,7 @@ The following functions provide locale-independent string to number conversions. If ``s`` represents a value that is too large to store in a float (for example, ``"1e500"`` is such a string on many platforms) then - if ``overflow_exception`` is ``NULL`` return ``Py_INFINITY`` (with + if ``overflow_exception`` is ``NULL`` return :c:macro:`!INFINITY` (with an appropriate sign) and don't set any exception. Otherwise, ``overflow_exception`` must point to a Python exception object; raise that exception and return ``-1.0``. In both cases, set @@ -128,22 +128,46 @@ The following functions provide locale-independent string to number conversions. must be 0 and is ignored. The ``'r'`` format code specifies the standard :func:`repr` format. - *flags* can be zero or more of the values ``Py_DTSF_SIGN``, - ``Py_DTSF_ADD_DOT_0``, or ``Py_DTSF_ALT``, or-ed together: + *flags* can be zero or more of the following values or-ed together: - * ``Py_DTSF_SIGN`` means to always precede the returned string with a sign - character, even if *val* is non-negative. + .. c:namespace:: NULL - * ``Py_DTSF_ADD_DOT_0`` means to ensure that the returned string will not look - like an integer. + .. c:macro:: Py_DTSF_SIGN - * ``Py_DTSF_ALT`` means to apply "alternate" formatting rules. See the - documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for - details. + Always precede the returned string with a sign + character, even if *val* is non-negative. - If *ptype* is non-``NULL``, then the value it points to will be set to one of - ``Py_DTST_FINITE``, ``Py_DTST_INFINITE``, or ``Py_DTST_NAN``, signifying that - *val* is a finite number, an infinite number, or not a number, respectively. + .. c:macro:: Py_DTSF_ADD_DOT_0 + + Ensure that the returned string will not look like an integer. + + .. c:macro:: Py_DTSF_ALT + + Apply "alternate" formatting rules. + See the documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for + details. + + .. c:macro:: Py_DTSF_NO_NEG_0 + + Negative zero is converted to positive zero. + + .. versionadded:: 3.11 + + If *ptype* is non-``NULL``, then the value it points to will be set to one + of the following constants depending on the type of *val*: + + .. list-table:: + :header-rows: 1 + :align: left + + * - *\*ptype* + - type of *val* + * - .. c:macro:: Py_DTST_FINITE + - finite number + * - .. c:macro:: Py_DTST_INFINITE + - infinite number + * - .. c:macro:: Py_DTST_NAN + - not a number The return value is a pointer to *buffer* with the converted string or ``NULL`` if the conversion failed. The caller is responsible for freeing the @@ -152,13 +176,85 @@ The following functions provide locale-independent string to number conversions. .. versionadded:: 3.1 -.. c:function:: int PyOS_stricmp(const char *s1, const char *s2) +.. c:function:: int PyOS_mystricmp(const char *str1, const char *str2) + int PyOS_mystrnicmp(const char *str1, const char *str2, Py_ssize_t size) + + Case insensitive comparison of strings. These functions work almost + identically to :c:func:`!strcmp` and :c:func:`!strncmp` (respectively), + except that they ignore the case of ASCII characters. + + Return ``0`` if the strings are equal, a negative value if *str1* sorts + lexicographically before *str2*, or a positive value if it sorts after. + + In the *str1* or *str2* arguments, a NUL byte marks the end of the string. + For :c:func:`!PyOS_mystrnicmp`, the *size* argument gives the maximum size + of the string, as if NUL was present at the index given by *size*. + + These functions do not use the locale. + + +.. c:function:: int PyOS_stricmp(const char *str1, const char *str2) + int PyOS_strnicmp(const char *str1, const char *str2, Py_ssize_t size) + + Case insensitive comparison of strings. + + On Windows, these are aliases of :c:func:`!stricmp` and :c:func:`!strnicmp`, + respectively. + + On other platforms, they are aliases of :c:func:`PyOS_mystricmp` and + :c:func:`PyOS_mystrnicmp`, respectively. + + +Character classification and conversion +======================================= + +The following macros provide locale-independent (unlike the C standard library +``ctype.h``) character classification and conversion. +The argument must be a signed or unsigned :c:expr:`char`. + + +.. c:macro:: Py_ISALNUM(c) + + Return true if the character *c* is an alphanumeric character. + + +.. c:macro:: Py_ISALPHA(c) + + Return true if the character *c* is an alphabetic character (``a-z`` and ``A-Z``). + + +.. c:macro:: Py_ISDIGIT(c) + + Return true if the character *c* is a decimal digit (``0-9``). + + +.. c:macro:: Py_ISLOWER(c) + + Return true if the character *c* is a lowercase ASCII letter (``a-z``). + + +.. c:macro:: Py_ISUPPER(c) + + Return true if the character *c* is an uppercase ASCII letter (``A-Z``). + + +.. c:macro:: Py_ISSPACE(c) + + Return true if the character *c* is a whitespace character (space, tab, + carriage return, newline, vertical tab, or form feed). + + +.. c:macro:: Py_ISXDIGIT(c) + + Return true if the character *c* is a hexadecimal digit (``0-9``, ``a-f``, and + ``A-F``). + + +.. c:macro:: Py_TOLOWER(c) - Case insensitive comparison of strings. The function works almost - identically to :c:func:`!strcmp` except that it ignores the case. + Return the lowercase equivalent of the character *c*. -.. c:function:: int PyOS_strnicmp(const char *s1, const char *s2, Py_ssize_t size) +.. c:macro:: Py_TOUPPER(c) - Case insensitive comparison of strings. The function works almost - identically to :c:func:`!strncmp` except that it ignores the case. + Return the uppercase equivalent of the character *c*. diff --git a/Doc/c-api/coro.rst b/Doc/c-api/coro.rst index caa855a10d20ca9..06422fb63f34f1a 100644 --- a/Doc/c-api/coro.rst +++ b/Doc/c-api/coro.rst @@ -33,3 +33,9 @@ return. with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. A reference to *frame* is stolen by this function. The *frame* argument must not be ``NULL``. + + .. deprecated-removed:: 3.16 3.18 + + This function has not been used since 3.10. + It is also impossible to construct a proper *frame* + object to call this function. diff --git a/Doc/c-api/curses.rst b/Doc/c-api/curses.rst new file mode 100644 index 000000000000000..5a1697c43cc969d --- /dev/null +++ b/Doc/c-api/curses.rst @@ -0,0 +1,138 @@ +.. highlight:: c + +Curses C API +------------ + +:mod:`curses` exposes a small C interface for extension modules. +Consumers must include the header file :file:`py_curses.h` (which is not +included by default by :file:`Python.h`) and :c:func:`import_curses` must +be invoked, usually as part of the module initialisation function, to populate +:c:var:`PyCurses_API`. + +.. warning:: + + Neither the C API nor the pure Python :mod:`curses` module are compatible + with subinterpreters. + +.. c:macro:: import_curses() + + Import the curses C API. The macro does not need a semi-colon to be called. + + On success, populate the :c:var:`PyCurses_API` pointer. + + On failure, set :c:var:`PyCurses_API` to NULL and set an exception. + The caller must check if an error occurred via :c:func:`PyErr_Occurred`: + + .. code-block:: + + import_curses(); // semi-colon is optional but recommended + if (PyErr_Occurred()) { /* cleanup */ } + + +.. c:var:: void **PyCurses_API + + Dynamically allocated object containing the curses C API. + This variable is only available once :c:macro:`import_curses` succeeds. + + ``PyCurses_API[0]`` corresponds to :c:data:`PyCursesWindow_Type`. + + ``PyCurses_API[1]``, ``PyCurses_API[2]``, and ``PyCurses_API[3]`` + are pointers to predicate functions of type ``int (*)(void)``. + + When called, these predicates return whether :func:`curses.setupterm`, + :func:`curses.initscr`, and :func:`curses.start_color` have been called + respectively. + + See also the convenience macros :c:macro:`PyCursesSetupTermCalled`, + :c:macro:`PyCursesInitialised`, and :c:macro:`PyCursesInitialisedColor`. + + .. note:: + + The number of entries in this structure is subject to changes. + Consider using :c:macro:`PyCurses_API_pointers` to check if + new fields are available or not. + + +.. c:macro:: PyCurses_API_pointers + + The number of accessible fields (``4``) in :c:var:`PyCurses_API`. + This number is incremented whenever new fields are added. + + +.. c:var:: PyTypeObject PyCursesWindow_Type + + The :ref:`heap type ` corresponding to :class:`curses.window`. + + +.. c:function:: int PyCursesWindow_Check(PyObject *op) + + Return true if *op* is a :class:`curses.window` instance, false otherwise. + + +The following macros are convenience macros expanding into C statements. +In particular, they can only be used as ``macro;`` or ``macro``, but not +``macro()`` or ``macro();``. + +.. c:macro:: PyCursesSetupTermCalled + + Macro checking if :func:`curses.setupterm` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_setupterm_called = (predicate_t)PyCurses_API[1]; + if (!was_setupterm_called()) { + return NULL; + } + } + + +.. c:macro:: PyCursesInitialised + + Macro checking if :func:`curses.initscr` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_initscr_called = (predicate_t)PyCurses_API[2]; + if (!was_initscr_called()) { + return NULL; + } + } + + +.. c:macro:: PyCursesInitialisedColor + + Macro checking if :func:`curses.start_color` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_start_color_called = (predicate_t)PyCurses_API[3]; + if (!was_start_color_called()) { + return NULL; + } + } + + +Internal data +------------- + +The following objects are exposed by the C API but should be considered +internal-only. + +.. c:macro:: PyCurses_CAPSULE_NAME + + Name of the curses capsule to pass to :c:func:`PyCapsule_Import`. + + Internal usage only. Use :c:macro:`import_curses` instead. + diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index d2d4d5309c70984..d7b4e116c49e355 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -8,11 +8,51 @@ DateTime Objects Various date and time objects are supplied by the :mod:`datetime` module. Before using any of these functions, the header file :file:`datetime.h` must be included in your source (note that this is not included by :file:`Python.h`), -and the macro :c:macro:`!PyDateTime_IMPORT` must be invoked, usually as part of +and the macro :c:macro:`PyDateTime_IMPORT` must be invoked, usually as part of the module initialisation function. The macro puts a pointer to a C structure -into a static variable, :c:data:`!PyDateTimeAPI`, that is used by the following +into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following macros. +.. c:macro:: PyDateTime_IMPORT() + + Import the datetime C API. + + On success, populate the :c:var:`PyDateTimeAPI` pointer. + On failure, set :c:var:`PyDateTimeAPI` to ``NULL`` and set an exception. + The caller must check if an error occurred via :c:func:`PyErr_Occurred`: + + .. code-block:: + + PyDateTime_IMPORT; + if (PyErr_Occurred()) { /* cleanup */ } + + .. warning:: + + This is not compatible with subinterpreters. + + .. versionchanged:: 3.15 + + This macro is now thread safe. + +.. c:type:: PyDateTime_CAPI + + Structure containing the fields for the datetime C API. + + The fields of this structure are private and subject to change. + + Do not use this directly; prefer ``PyDateTime_*`` APIs instead. + +.. c:var:: PyDateTime_CAPI *PyDateTimeAPI + + Dynamically allocated object containing the datetime C API. + + This variable is only available once :c:macro:`PyDateTime_IMPORT` succeeds. + + .. versionchanged:: 3.15 + + This variable should not be accessed directly as direct access is not thread-safe. + Use :c:func:`PyDateTime_IMPORT` instead. + .. c:type:: PyDateTime_Date This subtype of :c:type:`PyObject` represents a Python date object. @@ -46,7 +86,7 @@ macros. .. c:var:: PyTypeObject PyDateTime_DeltaType - This instance of :c:type:`PyTypeObject` represents Python type for + This instance of :c:type:`PyTypeObject` represents the Python type for the difference between two datetime values; it is the same object as :class:`datetime.timedelta` in the Python layer. @@ -325,3 +365,16 @@ Macros for the convenience of modules implementing the DB API: Create and return a new :class:`datetime.date` object given an argument tuple suitable for passing to :meth:`datetime.date.fromtimestamp`. + + +Internal data +------------- + +The following symbols are exposed by the C API but should be considered +internal-only. + +.. c:macro:: PyDateTime_CAPSULE_NAME + + Name of the datetime capsule to pass to :c:func:`PyCapsule_Import`. + + Internal usage only. Use :c:macro:`PyDateTime_IMPORT` instead. diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index b32c113e5f04573..539c4610ce4f5bc 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -8,33 +8,200 @@ Descriptor Objects "Descriptors" are objects that describe some attribute of an object. They are found in the dictionary of type objects. -.. XXX document these! +.. c:function:: PyObject* PyDescr_NewGetSet(PyTypeObject *type, struct PyGetSetDef *getset) -.. c:var:: PyTypeObject PyProperty_Type + Create a new get-set descriptor for extension type *type* from the + :c:type:`PyGetSetDef` structure *getset*. - The type object for the built-in descriptor types. + Get-set descriptors expose attributes implemented by C getter and setter + functions rather than stored directly in the instance. This is the same kind + of descriptor created for entries in :c:member:`~PyTypeObject.tp_getset`, and + it appears in Python as a :class:`types.GetSetDescriptorType` object. + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. -.. c:function:: PyObject* PyDescr_NewGetSet(PyTypeObject *type, struct PyGetSetDef *getset) +.. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *member) + + Create a new member descriptor for extension type *type* from the + :c:type:`PyMemberDef` structure *member*. + + Member descriptors expose fields in the type's C struct as Python + attributes. This is the same kind of descriptor created for entries in + :c:member:`~PyTypeObject.tp_members`, and it appears in Python as a + :class:`types.MemberDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. + +.. c:var:: PyTypeObject PyMemberDescr_Type + + The type object for member descriptor objects created from + :c:type:`PyMemberDef` structures. These descriptors expose fields of a + C struct as attributes on a type, and correspond + to :class:`types.MemberDescriptorType` objects in Python. -.. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *meth) + +.. c:var:: PyTypeObject PyGetSetDescr_Type + + The type object for get/set descriptor objects created from + :c:type:`PyGetSetDef` structures. These descriptors implement attributes + whose value is computed by C getter and setter functions, and are used + for many built-in type attributes. They correspond to + :class:`types.GetSetDescriptorType` objects in Python. .. c:function:: PyObject* PyDescr_NewMethod(PyTypeObject *type, struct PyMethodDef *meth) + Create a new method descriptor for extension type *type* from the + :c:type:`PyMethodDef` structure *meth*. + + Method descriptors expose C functions as methods on a type. This is the same + kind of descriptor created for entries in + :c:member:`~PyTypeObject.tp_methods`, and it appears in Python as a + :class:`types.MethodDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. + +.. c:var:: PyTypeObject PyMethodDescr_Type + + The type object for method descriptor objects created from + :c:type:`PyMethodDef` structures. These descriptors expose C functions as + methods on a type, and correspond to :class:`types.MethodDescriptorType` + objects in Python. + -.. c:function:: PyObject* PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *wrapper, void *wrapped) +.. c:struct:: wrapperbase + + Describes a slot wrapper used by :c:func:`PyDescr_NewWrapper`. + + Each ``wrapperbase`` record stores the Python-visible name and metadata for a + special method implemented by a type slot, together with the wrapper + function used to adapt that slot to Python's calling convention. + +.. c:function:: PyObject* PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *base, void *wrapped) + + Create a new wrapper descriptor for extension type *type* from the + :c:struct:`wrapperbase` structure *base* and the wrapped slot function + pointer + *wrapped*. + + Wrapper descriptors expose special methods implemented by type slots. This + is the same kind of descriptor that CPython creates for slot-based special + methods such as ``__repr__`` or ``__add__``, and it appears in Python as a + :class:`types.WrapperDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. + +.. c:var:: PyTypeObject PyWrapperDescr_Type + + The type object for wrapper descriptor objects created by + :c:func:`PyDescr_NewWrapper` and :c:func:`PyWrapper_New`. Wrapper + descriptors are used internally to expose special methods implemented + via wrapper structures, and appear in Python as + :class:`types.WrapperDescriptorType` objects. .. c:function:: PyObject* PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method) + Create a new class method descriptor for extension type *type* from the + :c:type:`PyMethodDef` structure *method*. + + Class method descriptors expose C methods that receive the class rather than + an instance when accessed. This is the same kind of descriptor created for + ``METH_CLASS`` entries in :c:member:`~PyTypeObject.tp_methods`, and it + appears in Python as a :class:`types.ClassMethodDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. .. c:function:: int PyDescr_IsData(PyObject *descr) - Return non-zero if the descriptor objects *descr* describes a data attribute, or + Return non-zero if the descriptor object *descr* describes a data attribute, or ``0`` if it describes a method. *descr* must be a descriptor object; there is no error checking. -.. c:function:: PyObject* PyWrapper_New(PyObject *, PyObject *) +.. c:function:: PyObject* PyWrapper_New(PyObject *d, PyObject *self) + + Create a new bound wrapper object from the wrapper descriptor *d* and the + instance *self*. + + This is the bound form of a wrapper descriptor created by + :c:func:`PyDescr_NewWrapper`. CPython creates these objects when a slot + wrapper is accessed through an instance, and they appear in Python as + :class:`types.MethodWrapperType` objects. + + On success, return a :term:`strong reference` to the wrapper object. Return + ``NULL`` with an exception set on failure. + +.. c:macro:: PyDescr_COMMON + + This is a macro including the common fields for a + descriptor object. + + This was included in Python's C API by mistake; do not use it in extensions. + For creating custom descriptor objects, create a class implementing the + descriptor protocol (:c:member:`~PyTypeObject.tp_descr_get` and + :c:member:`~PyTypeObject.tp_descr_set`). + + .. soft-deprecated:: 3.15 + + +Built-in descriptors +^^^^^^^^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyProperty_Type + + The type object for property objects. This is the same object as + :class:`property` in the Python layer. + + +.. c:var:: PyTypeObject PySuper_Type + + The type object for super objects. This is the same object as + :class:`super` in the Python layer. + + +.. c:var:: PyTypeObject PyClassMethod_Type + + The type of class method objects. This is the same object as + :class:`classmethod` in the Python layer. + + +.. c:var:: PyTypeObject PyClassMethodDescr_Type + + The type object for C-level class method descriptor objects. + This is the type of the descriptors created for :func:`classmethod` defined + in C extension types, and corresponds to + :class:`types.ClassMethodDescriptorType` objects in Python. + + +.. c:function:: PyObject *PyClassMethod_New(PyObject *callable) + + Create a new :class:`classmethod` object wrapping *callable*. + *callable* must be a callable object and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to a new class + method descriptor. On failure, this function returns ``NULL`` with an + exception set. + + +.. c:var:: PyTypeObject PyStaticMethod_Type + + The type of static method objects. This is the same object as + :class:`staticmethod` in the Python layer. + + +.. c:function:: PyObject *PyStaticMethod_New(PyObject *callable) + + Create a new :class:`staticmethod` object wrapping *callable*. + *callable* must be a callable object and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to a new static + method descriptor. On failure, this function returns ``NULL`` with an + exception set. + diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index e55c5c80cb83c0e..556113a97bf772f 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -2,7 +2,7 @@ .. _dictobjects: -Dictionary Objects +Dictionary objects ------------------ .. index:: pair: object; dictionary @@ -42,18 +42,48 @@ Dictionary Objects enforces read-only behavior. This is normally used to create a view to prevent modification of the dictionary for non-dynamic class types. + The first argument can be a :class:`dict`, a :class:`frozendict`, or a + mapping. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + + +.. c:var:: PyTypeObject PyDictProxy_Type + + The type object for mapping proxy objects created by + :c:func:`PyDictProxy_New` and for the read-only ``__dict__`` attribute + of many built-in types. A :c:type:`PyDictProxy_Type` instance provides a + dynamic, read-only view of an underlying dictionary: changes to the + underlying dictionary are reflected in the proxy, but the proxy itself + does not support mutation operations. This corresponds to + :class:`types.MappingProxyType` in Python. + .. c:function:: void PyDict_Clear(PyObject *p) Empty an existing dictionary of all key-value pairs. + Do nothing if the argument is not a :class:`dict` or a :class:`!dict` + subclass. + .. c:function:: int PyDict_Contains(PyObject *p, PyObject *key) - Determine if dictionary *p* contains *key*. If an item in *p* is matches + Determine if dictionary *p* contains *key*. If an item in *p* matches *key*, return ``1``, otherwise return ``0``. On error, return ``-1``. This is equivalent to the Python expression ``key in p``. + The first argument can be a :class:`dict` or a :class:`frozendict`. + + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: int PyDict_ContainsString(PyObject *p, const char *key) @@ -61,14 +91,18 @@ Dictionary Objects :c:expr:`const char*` UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. + The first argument can be a :class:`dict` or a :class:`frozendict`. + .. versionadded:: 3.13 + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: PyObject* PyDict_Copy(PyObject *p) Return a new dictionary that contains the same key-value pairs as *p*. - .. c:function:: int PyDict_SetItem(PyObject *p, PyObject *key, PyObject *val) Insert *val* into the dictionary *p* with a key of *key*. *key* must be @@ -76,6 +110,11 @@ Dictionary Objects ``0`` on success or ``-1`` on failure. This function *does not* steal a reference to *val*. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. c:function:: int PyDict_SetItemString(PyObject *p, const char *key, PyObject *val) @@ -91,6 +130,11 @@ Dictionary Objects If *key* is not in the dictionary, :exc:`KeyError` is raised. Return ``0`` on success or ``-1`` on failure. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. c:function:: int PyDict_DelItemString(PyObject *p, const char *key) @@ -107,10 +151,20 @@ Dictionary Objects * If the key is present, set *\*result* to a new :term:`strong reference` to the value and return ``1``. * If the key is missing, set *\*result* to ``NULL`` and return ``0``. - * On error, raise an exception and return ``-1``. + * On error, raise an exception, set *\*result* to ``NULL`` and return ``-1``. + + The first argument can be a :class:`dict` or a :class:`frozendict`. + + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. .. versionadded:: 3.13 + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + See also the :c:func:`PyObject_GetItem` function. @@ -120,16 +174,28 @@ Dictionary Objects has a key *key*. Return ``NULL`` if the key *key* is missing *without* setting an exception. + The first argument can be a :class:`dict` or a :class:`frozendict`. + .. note:: Exceptions that occur while this calls :meth:`~object.__hash__` and :meth:`~object.__eq__` methods are silently ignored. Prefer the :c:func:`PyDict_GetItemWithError` function instead. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_GetItemRef`, which + returns a :term:`strong reference`. + .. versionchanged:: 3.10 Calling this API without an :term:`attached thread state` had been allowed for historical reason. It is no longer allowed. + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: PyObject* PyDict_GetItemWithError(PyObject *p, PyObject *key) @@ -138,6 +204,16 @@ Dictionary Objects occurred. Return ``NULL`` **without** an exception set if the key wasn't present. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_GetItemRef`, which + returns a :term:`strong reference`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: PyObject* PyDict_GetItemString(PyObject *p, const char *key) @@ -153,6 +229,16 @@ Dictionary Objects Prefer using the :c:func:`PyDict_GetItemWithError` function with your own :c:func:`PyUnicode_FromString` *key* instead. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_GetItemStringRef`, + which returns a :term:`strong reference`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: int PyDict_GetItemStringRef(PyObject *p, const char *key, PyObject **result) @@ -162,6 +248,9 @@ Dictionary Objects .. versionadded:: 3.13 + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: PyObject* PyDict_SetDefault(PyObject *p, PyObject *key, PyObject *defaultobj) @@ -173,6 +262,14 @@ Dictionary Objects .. versionadded:: 3.4 + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_SetDefaultRef`, + which returns a :term:`strong reference`. + + .. c:function:: int PyDict_SetDefaultRef(PyObject *p, PyObject *key, PyObject *default_value, PyObject **result) @@ -192,13 +289,18 @@ Dictionary Objects These may refer to the same object: in that case you hold two separate references to it. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. versionadded:: 3.13 .. c:function:: int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result) Remove *key* from dictionary *p* and optionally return the removed value. - Do not raise :exc:`KeyError` if the key missing. + Do not raise :exc:`KeyError` if the key is missing. - If the key is present, set *\*result* to a new reference to the removed value if *result* is not ``NULL``, and return ``1``. @@ -207,7 +309,12 @@ Dictionary Objects - On error, raise an exception and return ``-1``. Similar to :meth:`dict.pop`, but without the default value and - not raising :exc:`KeyError` if the key missing. + not raising :exc:`KeyError` if the key is missing. + + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. .. versionadded:: 3.13 @@ -225,17 +332,32 @@ Dictionary Objects Return a :c:type:`PyListObject` containing all the items from the dictionary. + The first argument can be a :class:`dict` or a :class:`frozendict`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: PyObject* PyDict_Keys(PyObject *p) Return a :c:type:`PyListObject` containing all the keys from the dictionary. + The first argument can be a :class:`dict` or a :class:`frozendict`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: PyObject* PyDict_Values(PyObject *p) Return a :c:type:`PyListObject` containing all the values from the dictionary *p*. + The first argument can be a :class:`dict` or a :class:`frozendict`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: Py_ssize_t PyDict_Size(PyObject *p) @@ -244,6 +366,19 @@ Dictionary Objects Return the number of items in the dictionary. This is equivalent to ``len(p)`` on a dictionary. + The argument can be a :class:`dict` or a :class:`frozendict`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + + +.. c:function:: Py_ssize_t PyDict_GET_SIZE(PyObject *p) + + Similar to :c:func:`PyDict_Size`, but without error checking. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: int PyDict_Next(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) @@ -258,6 +393,8 @@ Dictionary Objects value represents offsets within the internal dictionary structure, and since the structure is sparse, the offsets are not consecutive. + The first argument can be a :class:`dict` or a :class:`frozendict`. + For example:: PyObject *key, *value; @@ -291,7 +428,7 @@ Dictionary Objects } The function is not thread-safe in the :term:`free-threaded ` - build without external synchronization. You can use + build without external synchronization for a mutable :class:`dict`. You can use :c:macro:`Py_BEGIN_CRITICAL_SECTION` to lock the dictionary while iterating over it:: @@ -301,6 +438,20 @@ Dictionary Objects } Py_END_CRITICAL_SECTION(); + The function is thread-safe on a :class:`frozendict`. + + .. note:: + + On the free-threaded build, this function can be used safely inside a + critical section. However, the references returned for *pkey* and *pvalue* + are :term:`borrowed ` and are only valid while the + critical section is held. If you need to use these objects outside the + critical section or when the critical section can be suspended, create a + :term:`strong reference ` (for example, using + :c:func:`Py_NewRef`). + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. .. c:function:: int PyDict_Merge(PyObject *a, PyObject *b, int override) @@ -311,6 +462,13 @@ Dictionary Objects only be added if there is not a matching key in *a*. Return ``0`` on success or ``-1`` if an exception was raised. + .. note:: + + In the :term:`free-threaded build`, when *b* is a + :class:`dict` (with the standard iterator), both *a* and *b* are locked + for the duration of the operation. When *b* is a non-dict mapping, only + *a* is locked; *b* may be concurrently modified by another thread. + .. c:function:: int PyDict_Update(PyObject *a, PyObject *b) @@ -320,6 +478,13 @@ Dictionary Objects argument has no "keys" attribute. Return ``0`` on success or ``-1`` if an exception was raised. + .. note:: + + In the :term:`free-threaded build`, when *b* is a + :class:`dict` (with the standard iterator), both *a* and *b* are locked + for the duration of the operation. When *b* is a non-dict mapping, only + *a* is locked; *b* may be concurrently modified by another thread. + .. c:function:: int PyDict_MergeFromSeq2(PyObject *a, PyObject *seq2, int override) @@ -335,6 +500,13 @@ Dictionary Objects if override or key not in a: a[key] = value + .. note:: + + In the :term:`free-threaded ` build, only *a* is locked. + The iteration over *seq2* is not synchronized; *seq2* may be concurrently + modified by another thread. + + .. c:function:: int PyDict_AddWatcher(PyDict_WatchCallback callback) Register *callback* as a dictionary watcher. Return a non-negative integer @@ -342,6 +514,13 @@ Dictionary Objects of error (e.g. no more watcher IDs available), return ``-1`` and set an exception. + .. note:: + + This function is not internally synchronized. In the + :term:`free-threaded ` build, callers should ensure no + concurrent calls to :c:func:`PyDict_AddWatcher` or + :c:func:`PyDict_ClearWatcher` are in progress. + .. versionadded:: 3.12 .. c:function:: int PyDict_ClearWatcher(int watcher_id) @@ -350,6 +529,13 @@ Dictionary Objects :c:func:`PyDict_AddWatcher`. Return ``0`` on success, ``-1`` on error (e.g. if the given *watcher_id* was never registered.) + .. note:: + + This function is not internally synchronized. In the + :term:`free-threaded ` build, callers should ensure no + concurrent calls to :c:func:`PyDict_AddWatcher` or + :c:func:`PyDict_ClearWatcher` are in progress. + .. versionadded:: 3.12 .. c:function:: int PyDict_Watch(int watcher_id, PyObject *dict) @@ -417,3 +603,189 @@ Dictionary Objects it before returning. .. versionadded:: 3.12 + + +Dictionary view objects +^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: int PyDictViewSet_Check(PyObject *op) + + Return true if *op* is a view of a set inside a dictionary. This is currently + equivalent to :c:expr:`PyDictKeys_Check(op) || PyDictItems_Check(op)`. This + function always succeeds. + + +.. c:var:: PyTypeObject PyDictKeys_Type + + Type object for a view of dictionary keys. In Python, this is the type of + the object returned by :meth:`dict.keys`. + + +.. c:function:: int PyDictKeys_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary keys view. This function + always succeeds. + + +.. c:var:: PyTypeObject PyDictValues_Type + + Type object for a view of dictionary values. In Python, this is the type of + the object returned by :meth:`dict.values`. + + +.. c:function:: int PyDictValues_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary values view. This function + always succeeds. + + +.. c:var:: PyTypeObject PyDictItems_Type + + Type object for a view of dictionary items. In Python, this is the type of + the object returned by :meth:`dict.items`. + + +.. c:function:: int PyDictItems_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary items view. This function + always succeeds. + + +Frozen dictionary objects +^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 3.15 + + +.. c:var:: PyTypeObject PyFrozenDict_Type + + This instance of :c:type:`PyTypeObject` represents the Python frozen + dictionary type. + This is the same object as :class:`frozendict` in the Python layer. + + +.. c:function:: int PyAnyDict_Check(PyObject *p) + + Return true if *p* is a :class:`dict` object, a :class:`frozendict` object, + or an instance of a subtype of the :class:`!dict` or :class:`!frozendict` + type. + This function always succeeds. + + +.. c:function:: int PyAnyDict_CheckExact(PyObject *p) + + Return true if *p* is a :class:`dict` object or a :class:`frozendict` object, + but not an instance of a subtype of the :class:`!dict` or + :class:`!frozendict` type. + This function always succeeds. + + +.. c:function:: int PyFrozenDict_Check(PyObject *p) + + Return true if *p* is a :class:`frozendict` object or an instance of a + subtype of the :class:`!frozendict` type. + This function always succeeds. + + +.. c:function:: int PyFrozenDict_CheckExact(PyObject *p) + + Return true if *p* is a :class:`frozendict` object, but not an instance of a + subtype of the :class:`!frozendict` type. + This function always succeeds. + + +.. c:function:: PyObject* PyFrozenDict_New(PyObject *iterable) + + Return a new :class:`frozendict` from an iterable, or ``NULL`` on failure + with an exception set. + + Create an empty dictionary if *iterable* is ``NULL``. + + +Ordered dictionaries +^^^^^^^^^^^^^^^^^^^^ + +Python's C API provides interface for :class:`collections.OrderedDict` from C. +Since Python 3.7, dictionaries are ordered by default, so there is usually +little need for these functions; prefer ``PyDict*`` where possible. + + +.. c:var:: PyTypeObject PyODict_Type + + Type object for ordered dictionaries. This is the same object as + :class:`collections.OrderedDict` in the Python layer. + + +.. c:function:: int PyODict_Check(PyObject *od) + + Return true if *od* is an ordered dictionary object or an instance of a + subtype of the :class:`~collections.OrderedDict` type. This function + always succeeds. + + +.. c:function:: int PyODict_CheckExact(PyObject *od) + + Return true if *od* is an ordered dictionary object, but not an instance of + a subtype of the :class:`~collections.OrderedDict` type. + This function always succeeds. + + +.. c:var:: PyTypeObject PyODictKeys_Type + + Analogous to :c:type:`PyDictKeys_Type` for ordered dictionaries. + + +.. c:var:: PyTypeObject PyODictValues_Type + + Analogous to :c:type:`PyDictValues_Type` for ordered dictionaries. + + +.. c:var:: PyTypeObject PyODictItems_Type + + Analogous to :c:type:`PyDictItems_Type` for ordered dictionaries. + + +.. c:function:: PyObject *PyODict_New(void) + + Return a new empty ordered dictionary, or ``NULL`` on failure. + + This is analogous to :c:func:`PyDict_New`. + + +.. c:function:: int PyODict_SetItem(PyObject *od, PyObject *key, PyObject *value) + + Insert *value* into the ordered dictionary *od* with a key of *key*. + Return ``0`` on success or ``-1`` with an exception set on failure. + + This is analogous to :c:func:`PyDict_SetItem`. + + +.. c:function:: int PyODict_DelItem(PyObject *od, PyObject *key) + + Remove the entry in the ordered dictionary *od* with key *key*. + Return ``0`` on success or ``-1`` with an exception set on failure. + + This is analogous to :c:func:`PyDict_DelItem`. + + +These are :term:`soft deprecated` aliases to ``PyDict`` APIs: + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * ``PyODict`` + * ``PyDict`` + * * .. c:macro:: PyODict_GetItem(od, key) + * :c:func:`PyDict_GetItem` + * * .. c:macro:: PyODict_GetItemWithError(od, key) + * :c:func:`PyDict_GetItemWithError` + * * .. c:macro:: PyODict_GetItemString(od, key) + * :c:func:`PyDict_GetItemString` + * * .. c:macro:: PyODict_Contains(od, key) + * :c:func:`PyDict_Contains` + * * .. c:macro:: PyODict_Size(od) + * :c:func:`PyDict_Size` + * * .. c:macro:: PyODict_SIZE(od) + * :c:func:`PyDict_GET_SIZE` diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 3ff4631a8e53c4a..82f594e11300a7d 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -309,6 +309,14 @@ For convenience, some of these functions will always return a .. versionadded:: 3.4 +.. c:function:: void PyErr_RangedSyntaxLocationObject(PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset) + + Similar to :c:func:`PyErr_SyntaxLocationObject`, but also sets the + *end_lineno* and *end_col_offset* information for the current exception. + + .. versionadded:: 3.10 + + .. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string @@ -331,6 +339,23 @@ For convenience, some of these functions will always return a use. +.. c:function:: PyObject *PyErr_ProgramTextObject(PyObject *filename, int lineno) + + Get the source line in *filename* at line *lineno*. *filename* should be a + Python :class:`str` object. + + On success, this function returns a Python string object with the found line. + On failure, this function returns ``NULL`` without an exception set. + + +.. c:function:: PyObject *PyErr_ProgramText(const char *filename, int lineno) + + Similar to :c:func:`PyErr_ProgramTextObject`, but *filename* is a + :c:expr:`const char *`, which is decoded with the + :term:`filesystem encoding and error handler`, instead of a + Python object reference. + + Issuing warnings ================ @@ -387,13 +412,22 @@ an error value). .. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) - Function similar to :c:func:`PyErr_WarnEx`, but use + Function similar to :c:func:`PyErr_WarnEx`, but uses :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is an ASCII-encoded string. .. versionadded:: 3.2 +.. c:function:: int PyErr_WarnExplicitFormat(PyObject *category, const char *filename, int lineno, const char *module, PyObject *registry, const char *format, ...) + + Similar to :c:func:`PyErr_WarnExplicit`, but uses + :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is + an ASCII-encoded string. + + .. versionadded:: 3.2 + + .. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) Function similar to :c:func:`PyErr_WarnFormat`, but *category* is @@ -639,28 +673,51 @@ Signal Handling single: SIGINT (C macro) single: KeyboardInterrupt (built-in exception) - This function interacts with Python's signal handling. + Handle external interruptions, such as signals or activating a debugger, + whose processing has been delayed until it is safe + to run Python code and/or raise exceptions. - If the function is called from the main thread and under the main Python - interpreter, it checks whether a signal has been sent to the processes - and if so, invokes the corresponding signal handler. If the :mod:`signal` - module is supported, this can invoke a signal handler written in Python. + For example, pressing :kbd:`Ctrl-C` causes a terminal to send the + :py:data:`signal.SIGINT` signal. + This function executes the corresponding Python signal handler, which, + by default, raises the :exc:`KeyboardInterrupt` exception. - The function attempts to handle all pending signals, and then returns ``0``. - However, if a Python signal handler raises an exception, the error - indicator is set and the function returns ``-1`` immediately (such that - other pending signals may not have been handled yet: they will be on the - next :c:func:`PyErr_CheckSignals()` invocation). + :c:func:`!PyErr_CheckSignals` should be called by long-running C code + frequently enough so that the response appears immediate to humans. - If the function is called from a non-main thread, or under a non-main - Python interpreter, it does nothing and returns ``0``. + Handlers invoked by this function currently include: - This function can be called by long-running C code that wants to - be interruptible by user requests (such as by pressing Ctrl-C). + - Signal handlers, including Python functions registered using + the :mod:`signal` module. - .. note:: - The default Python signal handler for :c:macro:`!SIGINT` raises the - :exc:`KeyboardInterrupt` exception. + Signal handlers are only run in the main thread of the main interpreter. + + (This is where the function got the name: originally, signals + were the only way to interrupt the interpreter.) + + - Running the garbage collector, if necessary. + + - Executing a pending :ref:`remote debugger ` script. + + - Raise the exception set by :c:func:`PyThreadState_SetAsyncExc`. + + If any handler raises an exception, immediately return ``-1`` with that + exception set. + Any remaining interruptions are left to be processed on the next + :c:func:`PyErr_CheckSignals()` invocation, if appropriate. + + If all handlers finish successfully, or there are no handlers to run, + return ``0``. + + .. versionchanged:: 3.12 + This function may now invoke the garbage collector. + + .. versionchanged:: 3.14 + This function may now execute a remote debugger script, if remote + debugging is enabled. + + .. versionchanged:: 3.15 + The exception set by :c:func:`PyThreadState_SetAsyncExc` is now raised. .. c:function:: void PyErr_SetInterrupt() @@ -759,9 +816,33 @@ Exception Classes Return :c:member:`~PyTypeObject.tp_name` of the exception class *ob*. +.. c:macro:: PyException_HEAD + + This is a macro including the base fields for an + exception object. + + This was included in Python's C API by mistake and is not designed for use + in extensions. For creating custom exception objects, use + :c:func:`PyErr_NewException` or otherwise create a class inheriting from + :c:data:`PyExc_BaseException`. + + .. soft-deprecated:: 3.15 + + Exception Objects ================= +.. c:function:: int PyExceptionInstance_Check(PyObject *op) + + Return true if *op* is an instance of :class:`BaseException`, false + otherwise. This function always succeeds. + + +.. c:macro:: PyExceptionInstance_Class(op) + + Equivalent to :c:func:`Py_TYPE(op) `. + + .. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) Return the traceback associated with the exception as a new reference, as @@ -939,6 +1020,9 @@ because the :ref:`call protocol ` takes care of recursion handling. be concatenated to the :exc:`RecursionError` message caused by the recursion depth limit. + .. seealso:: + The :c:func:`PyUnstable_ThreadState_SetStackProtection` function. + .. versionchanged:: 3.9 This function is now also available in the :ref:`limited API `. @@ -954,7 +1038,7 @@ Properly implementing :c:member:`~PyTypeObject.tp_repr` for container types requ special recursion handling. In addition to protecting the stack, :c:member:`~PyTypeObject.tp_repr` also needs to track objects to prevent cycles. The following two functions facilitate this functionality. Effectively, -these are the C equivalent to :func:`reprlib.recursive_repr`. +these are the C equivalent to :deco:`reprlib.recursive_repr`. .. c:function:: int Py_ReprEnter(PyObject *object) @@ -979,6 +1063,27 @@ these are the C equivalent to :func:`reprlib.recursive_repr`. Ends a :c:func:`Py_ReprEnter`. Must be called once for each invocation of :c:func:`Py_ReprEnter` that returns zero. +.. c:function:: int Py_GetRecursionLimit(void) + + Get the recursion limit for the current interpreter. It can be set with + :c:func:`Py_SetRecursionLimit`. The recursion limit prevents the + Python interpreter stack from growing infinitely. + + This function cannot fail, and the caller must hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`sys.getrecursionlimit` + +.. c:function:: void Py_SetRecursionLimit(int new_limit) + + Set the recursion limit for the current interpreter. + + This function cannot fail, and the caller must hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`sys.setrecursionlimit` .. _standardexceptions: @@ -1039,6 +1144,8 @@ Exception types * :exc:`FloatingPointError` * * .. c:var:: PyObject *PyExc_GeneratorExit * :exc:`GeneratorExit` + * * .. c:var:: PyObject *PyExc_ImportCycleError + * :exc:`ImportCycleError` * * .. c:var:: PyObject *PyExc_ImportError * :exc:`ImportError` * * .. c:var:: PyObject *PyExc_IndentationError @@ -1207,3 +1314,101 @@ Warning types .. versionadded:: 3.10 :c:data:`PyExc_EncodingWarning`. + + +Tracebacks +========== + +.. c:var:: PyTypeObject PyTraceBack_Type + + Type object for traceback objects. This is available as + :class:`types.TracebackType` in the Python layer. + + +.. c:function:: int PyTraceBack_Check(PyObject *op) + + Return true if *op* is a traceback object, false otherwise. This function + does not account for subtypes. + + +.. c:function:: int PyTraceBack_Here(PyFrameObject *f) + + Replace the :attr:`~BaseException.__traceback__` attribute on the current + exception with a new traceback prepending *f* to the existing chain. + + Calling this function without an exception set is undefined behavior. + + This function returns ``0`` on success, and returns ``-1`` with an + exception set on failure. + + +.. c:function:: int PyTraceBack_Print(PyObject *tb, PyObject *f) + + Write the traceback *tb* into the file *f*. + + This function returns ``0`` on success, and returns ``-1`` with an + exception set on failure. + +.. c:function:: const char* PyUnstable_DumpTraceback(int fd, PyThreadState *tstate) + + Write a trace of the Python stack in *tstate* into the file *fd*. The format + looks like:: + + Traceback (most recent call first): + File "xxx", line xxx in + File "xxx", line xxx in + ... + File "xxx", line xxx in + + This function is meant to debug situations such as segfaults, fatal errors, + and similar. The file and function names it outputs are encoded to ASCII with + backslashreplace and truncated to 500 characters. It writes only the first + 100 frames; further frames are truncated with the line ``...``. + + This function will return ``NULL`` on success, or an error message on error. + + This function is intended for use in crash scenarios such as signal handlers + for SIGSEGV, where the interpreter may be in an inconsistent state. Given + that it reads interpreter data structures that may be partially modified, the + function might produce incomplete output or it may even crash itself. + + The caller does not need to hold an :term:`attached thread state`, nor does + *tstate* need to be attached. + + .. versionadded:: 3.15 + +.. c:function:: const char* PyUnstable_DumpTracebackThreads(int fd, PyInterpreterState *interp, PyThreadState *current_tstate, Py_ssize_t max_threads) + + Write the traces of all Python threads in *interp* into the file *fd*. + + If *interp* is ``NULL`` then this function will try to identify the current + interpreter using thread-specific storage. If it cannot, it will return an + error. + + If *current_tstate* is not ``NULL`` then it will be used to identify what the + current thread is in the written output. If it is ``NULL`` then this function + will identify the current thread using thread-specific storage. It is not an + error if the function is unable to get the current Python thread state. + + This function will return ``NULL`` on success, or an error message on error. + + This function is meant to debug situations such as segfaults, fatal + errors, and similar. It calls :c:func:`PyUnstable_DumpTraceback` for each + thread. It only writes the tracebacks of the first *max_threads* threads, + further output is truncated with the line ``...``. If *max_threads* is 0, the + function will use a default value of 100 for the argument. + + This function is intended for use in crash scenarios such as signal handlers + for SIGSEGV, where the interpreter may be in an inconsistent state. Given + that it reads interpreter data structures that may be partially modified, the + function might produce incomplete output or it may even crash itself. + + The caller does not need to hold an :term:`attached thread state`, nor does + *current_tstate* need to be attached. + + .. warning:: + On the :term:`free-threaded build`, this function is not thread-safe. If + another thread deletes its :term:`thread state` while this function is being + called, the process will likely crash. + + .. versionadded:: 3.15 diff --git a/Doc/c-api/extension-modules.rst b/Doc/c-api/extension-modules.rst index 3d331e6ec12f76e..34ee86c7876ae74 100644 --- a/Doc/c-api/extension-modules.rst +++ b/Doc/c-api/extension-modules.rst @@ -8,7 +8,8 @@ Defining extension modules A C extension for CPython is a shared library (for example, a ``.so`` file on Linux, ``.pyd`` DLL on Windows), which is loadable into the Python process (for example, it is compiled with compatible compiler settings), and which -exports an :ref:`initialization function `. +exports an :dfn:`export hook` function (or an +old-style :ref:`initialization function `). To be importable by default (that is, by :py:class:`importlib.machinery.ExtensionFileLoader`), @@ -23,25 +24,127 @@ and must be named after the module name plus an extension listed in One suitable tool is Setuptools, whose documentation can be found at https://setuptools.pypa.io/en/latest/setuptools.html. -Normally, the initialization function returns a module definition initialized -using :c:func:`PyModuleDef_Init`. -This allows splitting the creation process into several phases: +.. _extension-export-hook: + +Extension export hook +..................... + +.. versionadded:: 3.15 + + Support for the :samp:`PyModExport_{}` export hook was added in Python + 3.15. The older way of defining modules is still available: consult either + the :ref:`extension-pyinit` section or earlier versions of this + documentation if you plan to support earlier Python versions. + +The export hook must be an exported function with the following signature: + +.. c:function:: PySlot *PyModExport_modulename(void) + +For modules with ASCII-only names, the :ref:`export hook ` +must be named :samp:`PyModExport_{}`, +with ```` replaced by the module's name. + +For non-ASCII module names, the export hook must instead be named +:samp:`PyModExportU_{}` (note the ``U``), with ```` encoded using +Python's *punycode* encoding with hyphens replaced by underscores. In Python: + +.. code-block:: python + + def hook_name(name): + try: + suffix = b'_' + name.encode('ascii') + except UnicodeEncodeError: + suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') + return b'PyModExport' + suffix + +The export hook returns an array of :c:type:`PySlot` entries, +terminated by an entry with a slot ID of ``0``. +These slots describe how the module should be created and initialized. + +This array must remain valid and constant until interpreter shutdown. +Typically, it should use ``static`` storage. +Prefer using the :c:macro:`Py_mod_create` and :c:macro:`Py_mod_exec` slots +for any dynamic behavior. + +The export hook may return ``NULL`` with an exception set to signal failure. + +It is recommended to define the export hook function using a helper macro: + +.. c:macro:: PyMODEXPORT_FUNC + + Declare an extension module export hook. + This macro: + + * specifies the :c:expr:`PySlot*` return type, + * adds any special linkage declarations required by the platform, and + * for C++, declares the function as ``extern "C"``. +For example, a module called ``spam`` would be defined like this:: + + PyABIInfo_VAR(abi_info); + + static PySlot spam_slots[] = { + PySlot_STATIC_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_name, "spam"), + PySlot_FUNC(Py_mod_init, spam_init_function), + ... + PySlot_END + }; + + PyMODEXPORT_FUNC + PyModExport_spam(void) + { + return spam_slots; + } + +The export hook is typically the only non-\ ``static`` +item defined in the module's C source. + +The hook should be kept short -- ideally, one line as above. +If you do need to use Python C API in this function, it is recommended to call +``PyABIInfo_Check(&abi_info, "modulename")`` first to raise an exception, +rather than crash, in common cases of ABI mismatch. + + +.. note:: + + It is possible to export multiple modules from a single shared library by + defining multiple export hooks. + However, importing them requires a custom importer or suitably named + copies/links of the extension file, because Python's import machinery only + finds the function corresponding to the filename. + See the `Multiple modules in one library `__ + section in :pep:`489` for details. + + +.. _multi-phase-initialization: + +Multi-phase initialization +.......................... + +The process of creating an extension module follows several phases: + +- Python finds and calls the export hook to get information on how to + create the module. - Before any substantial code is executed, Python can determine which capabilities the module supports, and it can adjust the environment or refuse loading an incompatible extension. -- By default, Python itself creates the module object -- that is, it does - the equivalent of :py:meth:`object.__new__` for classes. - It also sets initial attributes like :attr:`~module.__package__` and - :attr:`~module.__loader__`. -- Afterwards, the module object is initialized using extension-specific - code -- the equivalent of :py:meth:`~object.__init__` on classes. + Slots like :c:data:`Py_mod_abi`, :c:data:`Py_mod_gil` and + :c:data:`Py_mod_multiple_interpreters` influence this step. +- By default, Python itself then creates the module object -- that is, it does + the equivalent of calling :py:meth:`~object.__new__` when creating an object. + This step can be overridden using the :c:data:`Py_mod_create` slot. +- Python sets initial module attributes like :attr:`~module.__package__` and + :attr:`~module.__loader__`, and inserts the module object into + :py:attr:`sys.modules`. +- Afterwards, the module object is initialized in an extension-specific way + -- the equivalent of :py:meth:`~object.__init__` when creating an object, + or of executing top-level code in a Python-language module. + The behavior is specified using the :c:data:`Py_mod_exec` slot. This is called *multi-phase initialization* to distinguish it from the legacy -(but still supported) *single-phase initialization* scheme, -where the initialization function returns a fully constructed module. -See the :ref:`single-phase-initialization section below ` -for details. +(but still supported) :ref:`single-phase initialization `, +where an initialization function returns a fully constructed module. .. versionchanged:: 3.5 @@ -53,7 +156,7 @@ Multiple module instances By default, extension modules are not singletons. For example, if the :py:attr:`sys.modules` entry is removed and the module -is re-imported, a new module object is created, and typically populated with +is re-imported, a new module object is created and, typically, populated with fresh method and type objects. The old module is subject to normal garbage collection. This mirrors the behavior of pure-Python modules. @@ -83,36 +186,34 @@ A module may also be limited to the main interpreter using the :c:data:`Py_mod_multiple_interpreters` slot. -.. _extension-export-hook: +.. _extension-pyinit: -Initialization function -....................... +``PyInit`` function +................... -The initialization function defined by an extension module has the -following signature: +.. soft-deprecated:: 3.15 + + This functionality will not get new features, + but there are no plans to remove it. + +Instead of :c:func:`PyModExport_modulename`, an extension module can define +an older-style :dfn:`initialization function` with the signature: .. c:function:: PyObject* PyInit_modulename(void) Its name should be :samp:`PyInit_{}`, with ```` replaced by the name of the module. +For non-ASCII module names, use :samp:`PyInitU_{}` instead, with +```` encoded in the same way as for the +:ref:`export hook ` (that is, using Punycode +with underscores). -For modules with ASCII-only names, the function must instead be named -:samp:`PyInit_{}`, with ```` replaced by the name of the module. -When using :ref:`multi-phase-initialization`, non-ASCII module names -are allowed. In this case, the initialization function name is -:samp:`PyInitU_{}`, with ```` encoded using Python's -*punycode* encoding with hyphens replaced by underscores. In Python: +If a module exports both :samp:`PyInit_{}` and +:samp:`PyModExport_{}`, the :samp:`PyInit_{}` function +is ignored. -.. code-block:: python - - def initfunc_name(name): - try: - suffix = b'_' + name.encode('ascii') - except UnicodeEncodeError: - suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') - return b'PyInit' + suffix - -It is recommended to define the initialization function using a helper macro: +Like with :c:macro:`PyMODEXPORT_FUNC`, it is recommended to define the +initialization function using a helper macro: .. c:macro:: PyMODINIT_FUNC @@ -123,43 +224,15 @@ It is recommended to define the initialization function using a helper macro: * adds any special linkage declarations required by the platform, and * for C++, declares the function as ``extern "C"``. -For example, a module called ``spam`` would be defined like this:: - - static struct PyModuleDef spam_module = { - .m_base = PyModuleDef_HEAD_INIT, - .m_name = "spam", - ... - }; - - PyMODINIT_FUNC - PyInit_spam(void) - { - return PyModuleDef_Init(&spam_module); - } - -It is possible to export multiple modules from a single shared library by -defining multiple initialization functions. However, importing them requires -using symbolic links or a custom importer, because by default only the -function corresponding to the filename is found. -See the `Multiple modules in one library `__ -section in :pep:`489` for details. - -The initialization function is typically the only non-\ ``static`` -item defined in the module's C source. +Normally, the initialization function (``PyInit_modulename``) returns +a :c:type:`PyModuleDef` instance with non-``NULL`` +:c:member:`~PyModuleDef.m_slots`. This allows Python to use +:ref:`multi-phase initialization `. -.. _multi-phase-initialization: - -Multi-phase initialization -.......................... - -Normally, the :ref:`initialization function ` -(``PyInit_modulename``) returns a :c:type:`PyModuleDef` instance with -non-``NULL`` :c:member:`~PyModuleDef.m_slots`. Before it is returned, the ``PyModuleDef`` instance must be initialized using the following function: - .. c:function:: PyObject* PyModuleDef_Init(PyModuleDef *def) Ensure a module definition is a properly initialized Python object that @@ -167,7 +240,8 @@ using the following function: Return *def* cast to ``PyObject*``, or ``NULL`` if an error occurred. - Calling this function is required for :ref:`multi-phase-initialization`. + Calling this function is required before returning a :c:type:`PyModuleDef` + from a module initialization function. It should not be used in other contexts. Note that Python assumes that ``PyModuleDef`` structures are statically @@ -178,18 +252,36 @@ using the following function: .. versionadded:: 3.5 +For example, a module called ``spam`` would be defined like this:: + + static struct PyModuleDef spam_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "spam", + ... + }; + + PyMODINIT_FUNC + PyInit_spam(void) + { + return PyModuleDef_Init(&spam_module); + } + + .. _single-phase-initialization: Legacy single-phase initialization -.................................. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. soft-deprecated:: 3.15 -.. attention:: Single-phase initialization is a legacy mechanism to initialize extension modules, with known drawbacks and design flaws. Extension module authors are encouraged to use multi-phase initialization instead. -In single-phase initialization, the -:ref:`initialization function ` (``PyInit_modulename``) + However, there are no plans to remove support for it. + +In single-phase initialization, the old-style +:ref:`initialization function ` (``PyInit_modulename``) should create, populate and return a module object. This is typically done using :c:func:`PyModule_Create` and functions like :c:func:`PyModule_AddObjectRef`. @@ -242,6 +334,8 @@ in the following ways: * Single-phase modules support module lookup functions like :c:func:`PyState_FindModule`. +* The module's :c:member:`PyModuleDef.m_slots` must be NULL. + .. [#testsinglephase] ``_testsinglephase`` is an internal module used in CPython's self-test suite; your installation may or may not include it. diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index e9019a0d500f7e5..dcafefdc0458722 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -2,7 +2,7 @@ .. _fileobjects: -File Objects +File objects ------------ .. index:: pair: object; file @@ -93,6 +93,29 @@ the :mod:`io` APIs instead. .. versionadded:: 3.8 +.. c:function:: PyObject *PyFile_OpenCodeObject(PyObject *path) + + Open *path* with the mode ``'rb'``. *path* must be a Python :class:`str` + object. The behavior of this function may be overridden by + :c:func:`PyFile_SetOpenCodeHook` to allow for some preprocessing of the + text. + + This is analogous to :func:`io.open_code` in Python. + + On success, this function returns a :term:`strong reference` to a Python + file object. On failure, this function returns ``NULL`` with an exception + set. + + .. versionadded:: 3.8 + + +.. c:function:: PyObject *PyFile_OpenCode(const char *path) + + Similar to :c:func:`PyFile_OpenCodeObject`, but *path* is a + UTF-8 encoded :c:expr:`const char*`. + + .. versionadded:: 3.8 + .. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags) @@ -100,11 +123,34 @@ the :mod:`io` APIs instead. Write object *obj* to file object *p*. The only supported flag for *flags* is :c:macro:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written - instead of the :func:`repr`. Return ``0`` on success or ``-1`` on failure; the - appropriate exception will be set. + instead of the :func:`repr`. + + If *obj* is ``NULL``, write the string ``""``. + Return ``0`` on success or ``-1`` on failure; the + appropriate exception will be set. .. c:function:: int PyFile_WriteString(const char *s, PyObject *p) Write string *s* to file object *p*. Return ``0`` on success or ``-1`` on failure; the appropriate exception will be set. + + +Soft-deprecated API +^^^^^^^^^^^^^^^^^^^ + +.. soft-deprecated:: 3.15 + +These are APIs that were included in Python's C API +by mistake. They are documented solely for completeness; use other +``PyFile*`` APIs instead. + +.. c:function:: PyObject *PyFile_NewStdPrinter(int fd) + + Use :c:func:`PyFile_FromFd` with defaults (``fd, NULL, "w", -1, NULL, NULL, NULL, 0``) instead. + +.. c:var:: PyTypeObject PyStdPrinter_Type + + Type of file-like objects used internally at Python startup when :py:mod:`io` is + not yet available. + Use Python :py:func:`open` or :c:func:`PyFile_FromFd` to create file objects instead. diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 489676caa3a16a8..a12ad11abb107d0 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -78,6 +78,109 @@ Floating-Point Objects Return the minimum normalized positive float *DBL_MIN* as C :c:expr:`double`. +.. c:macro:: Py_INFINITY + + This macro expands to a constant expression of type :c:expr:`double`, that + represents the positive infinity. + + It is equivalent to the :c:macro:`!INFINITY` macro from the C11 standard + ```` header. + + .. soft-deprecated:: 3.15 + + +.. c:macro:: Py_NAN + + This macro expands to a constant expression of type :c:expr:`double`, that + represents a quiet not-a-number (qNaN) value. + + On most platforms, this is equivalent to the :c:macro:`!NAN` macro from + the C11 standard ```` header. + + +.. c:macro:: Py_HUGE_VAL + + Equivalent to :c:macro:`!INFINITY`. + + .. soft-deprecated:: 3.14 + + +.. c:macro:: Py_MATH_E + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.e` constant. + + +.. c:macro:: Py_MATH_El + + High precision (long double) definition of :data:`~math.e` constant. + + .. deprecated-removed:: 3.15 3.20 + + +.. c:macro:: Py_MATH_PI + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.pi` constant. + + +.. c:macro:: Py_MATH_PIl + + High precision (long double) definition of :data:`~math.pi` constant. + + .. deprecated-removed:: 3.15 3.20 + + +.. c:macro:: Py_MATH_TAU + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.tau` constant. + + .. versionadded:: 3.6 + + +.. c:macro:: Py_RETURN_NAN + + Return :data:`math.nan` from a function. + + On most platforms, this is equivalent to ``return PyFloat_FromDouble(NAN)``. + + +.. c:macro:: Py_RETURN_INF(sign) + + Return :data:`math.inf` or :data:`-math.inf ` from a function, + depending on the sign of *sign*. + + On most platforms, this is equivalent to the following:: + + return PyFloat_FromDouble(copysign(INFINITY, sign)); + + +.. c:macro:: Py_IS_FINITE(X) + + Return ``1`` if the given floating-point number *X* is finite, + that is, it is normal, subnormal or zero, but not infinite or NaN. + Return ``0`` otherwise. + + .. soft-deprecated:: 3.14 + Use :c:macro:`!isfinite` instead. + + +.. c:macro:: Py_IS_INFINITY(X) + + Return ``1`` if the given floating-point number *X* is positive or negative + infinity. Return ``0`` otherwise. + + .. soft-deprecated:: 3.14 + Use :c:macro:`!isinf` instead. + + +.. c:macro:: Py_IS_NAN(X) + + Return ``1`` if the given floating-point number *X* is a not-a-number (NaN) + value. Return ``0`` otherwise. + + .. soft-deprecated:: 3.14 + Use :c:macro:`!isnan` instead. + + Pack and Unpack functions ------------------------- @@ -85,24 +188,23 @@ The pack and unpack functions provide an efficient platform-independent way to store floating-point values as byte strings. The Pack routines produce a bytes string from a C :c:expr:`double`, and the Unpack routines produce a C :c:expr:`double` from such a bytes string. The suffix (2, 4 or 8) specifies the -number of bytes in the bytes string. +number of bytes in the bytes string: -On platforms that appear to use IEEE 754 formats these functions work by -copying bits. On other platforms, the 2-byte format is identical to the IEEE -754 binary16 half-precision format, the 4-byte format (32-bit) is identical to -the IEEE 754 binary32 single precision format, and the 8-byte format to the -IEEE 754 binary64 double precision format, although the packing of INFs and -NaNs (if such things exist on the platform) isn't handled correctly, and -attempting to unpack a bytes string containing an IEEE INF or NaN will raise an -exception. +* The 2-byte format is the IEEE 754 binary16 half-precision format. +* The 4-byte format is the IEEE 754 binary32 single-precision format. +* The 8-byte format is the IEEE 754 binary64 double-precision format. -Note that NaNs type may not be preserved on IEEE platforms (silent NaN become -quiet), for example on x86 systems in 32-bit mode. +The NaN type may not be preserved on some platforms while unpacking (signaling +NaNs become quiet NaNs), for example on x86 systems in 32-bit mode. +It's assumed that the :c:expr:`double` type has the IEEE 754 binary64 double +precision format. What happens if it's not true is partly accidental (alas). On non-IEEE platforms with more precision, or larger dynamic range, than IEEE 754 supports, not all values can be packed; on non-IEEE platforms with less -precision, or smaller dynamic range, not all values can be unpacked. What -happens in such cases is partly accidental (alas). +precision, or smaller dynamic range, not all values can be unpacked. The +packing of special numbers like INFs and NaNs (if such things exist on the +platform) may not be handled correctly, and attempting to unpack a bytes string +containing an IEEE INF or NaN may raise an exception. .. versionadded:: 3.11 @@ -111,19 +213,14 @@ Pack functions The pack routines write 2, 4 or 8 bytes, starting at *p*. *le* is an :c:expr:`int` argument, non-zero if you want the bytes string in little-endian -format (exponent last, at ``p+1``, ``p+3``, or ``p+6`` ``p+7``), zero if you -want big-endian format (exponent first, at *p*). The :c:macro:`PY_BIG_ENDIAN` -constant can be used to use the native endian: it is equal to ``1`` on big -endian processor, or ``0`` on little endian processor. +format (exponent last, at ``p+1``, ``p+3``, or ``p+6`` and ``p+7``), zero if you +want big-endian format (exponent first, at *p*). Use the :c:macro:`!PY_LITTLE_ENDIAN` +constant to select the native endian: it is equal to ``0`` on big +endian processor, or ``1`` on little endian processor. Return value: ``0`` if all is OK, ``-1`` if error (and an exception is set, most likely :exc:`OverflowError`). -There are two problems on non-IEEE platforms: - -* What this does is undefined if *x* is a NaN or infinity. -* ``-0.0`` and ``+0.0`` produce the same bytes string. - .. c:function:: int PyFloat_Pack2(double x, char *p, int le) Pack a C double as the IEEE 754 binary16 half-precision format. @@ -136,6 +233,9 @@ There are two problems on non-IEEE platforms: Pack a C double as the IEEE 754 binary64 double precision format. + .. impl-detail:: + This function always succeeds in CPython. + Unpack functions ^^^^^^^^^^^^^^^^ @@ -143,16 +243,16 @@ Unpack functions The unpack routines read 2, 4 or 8 bytes, starting at *p*. *le* is an :c:expr:`int` argument, non-zero if the bytes string is in little-endian format (exponent last, at ``p+1``, ``p+3`` or ``p+6`` and ``p+7``), zero if big-endian -(exponent first, at *p*). The :c:macro:`PY_BIG_ENDIAN` constant can be used to -use the native endian: it is equal to ``1`` on big endian processor, or ``0`` +(exponent first, at *p*). Use the :c:macro:`!PY_LITTLE_ENDIAN` constant to +select the native endian: it is equal to ``0`` on big endian processor, or ``1`` on little endian processor. Return value: The unpacked double. On error, this is ``-1.0`` and :c:func:`PyErr_Occurred` is true (and an exception is set, most likely :exc:`OverflowError`). -Note that on a non-IEEE platform this will refuse to unpack a bytes string that -represents a NaN or infinity. +.. impl-detail:: + These functions always succeed in CPython. .. c:function:: double PyFloat_Unpack2(const char *p, int le) diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 1a52e146a697510..4159ff6e5965fbd 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -1,6 +1,6 @@ .. highlight:: c -Frame Objects +Frame objects ------------- .. c:type:: PyFrameObject @@ -29,6 +29,12 @@ See also :ref:`Reflection `. Previously, this type was only available after including ````. +.. c:function:: PyFrameObject *PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals, PyObject *locals) + + Create a new frame object. This function returns a :term:`strong reference` + to the new frame object on success, and returns ``NULL`` with an exception + set on failure. + .. c:function:: int PyFrame_Check(PyObject *obj) Return non-zero if *obj* is a frame object. @@ -44,6 +50,7 @@ See also :ref:`Reflection `. Return a :term:`strong reference`, or ``NULL`` if *frame* has no outer frame. + This raises no exceptions. .. versionadded:: 3.9 @@ -140,7 +147,7 @@ See also :ref:`Reflection `. Return the line number that *frame* is currently executing. -Frame Locals Proxies +Frame locals proxies ^^^^^^^^^^^^^^^^^^^^ .. versionadded:: 3.13 @@ -161,7 +168,52 @@ See :pep:`667` for more information. Return non-zero if *obj* is a frame :func:`locals` proxy. -Internal Frames + +Legacy local variable APIs +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These APIs are :term:`soft deprecated`. As of Python 3.13, they do nothing. +They exist solely for backwards compatibility. + + +.. c:function:: void PyFrame_LocalsToFast(PyFrameObject *f, int clear) + + Prior to Python 3.13, this function would copy the :attr:`~frame.f_locals` + attribute of *f* to the internal "fast" array of local variables, allowing + changes in frame objects to be visible to the interpreter. If *clear* was + true, this function would process variables that were unset in the locals + dictionary. + + .. soft-deprecated:: 3.13 + This function now does nothing. + + +.. c:function:: void PyFrame_FastToLocals(PyFrameObject *f) + + Prior to Python 3.13, this function would copy the internal "fast" array + of local variables (which is used by the interpreter) to the + :attr:`~frame.f_locals` attribute of *f*, allowing changes in local + variables to be visible to frame objects. + + .. soft-deprecated:: 3.13 + This function now does nothing. + + +.. c:function:: int PyFrame_FastToLocalsWithError(PyFrameObject *f) + + Prior to Python 3.13, this function was similar to + :c:func:`PyFrame_FastToLocals`, but would return ``0`` on success, and + ``-1`` with an exception set on failure. + + .. soft-deprecated:: 3.13 + This function now does nothing. + + +.. seealso:: + :pep:`667` + + +Internal frames ^^^^^^^^^^^^^^^ Unless using :pep:`523`, you will not need this. @@ -191,5 +243,3 @@ Unless using :pep:`523`, you will not need this. Return the currently executing line number, or -1 if there is no line number. .. versionadded:: 3.12 - - diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 5fb8567ef8c95fb..609b5e885b6b162 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -102,6 +102,15 @@ There are a few functions specific to Python functions. dictionary of arguments or ``NULL``. +.. c:function:: int PyFunction_SetKwDefaults(PyObject *op, PyObject *defaults) + + Set the keyword-only argument default values of the function object *op*. + *defaults* must be a dictionary of keyword-only arguments or ``Py_None``. + + This function returns ``0`` on success, and returns ``-1`` with an exception + set on failure. + + .. c:function:: PyObject* PyFunction_GetClosure(PyObject *op) Return the closure associated with the function object *op*. This can be ``NULL`` @@ -175,6 +184,9 @@ There are a few functions specific to Python functions. .. versionadded:: 3.12 + - ``PyFunction_PYFUNC_EVENT_MODIFY_QUALNAME`` + + .. versionadded:: 3.15 .. c:type:: int (*PyFunction_WatchCallback)(PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value) @@ -197,7 +209,7 @@ There are a few functions specific to Python functions. runtime behavior depending on optimization decisions, it does not change the semantics of the Python code being executed. - If *event* is ``PyFunction_EVENT_DESTROY``, Taking a reference in the + If *event* is ``PyFunction_EVENT_DESTROY``, taking a reference in the callback to the about-to-be-destroyed function will resurrect it, preventing it from being freed at this time. When the resurrected object is destroyed later, any watcher callbacks active at that time will be called again. diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index f6fa52b36c5ab31..9c71bb3d59d5e99 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -220,50 +220,237 @@ The :c:member:`~PyTypeObject.tp_traverse` handler accepts a function parameter o detection; it's not expected that users will need to write their own visitor functions. -The :c:member:`~PyTypeObject.tp_traverse` handler must have the following type: +The :c:member:`~PyTypeObject.tp_clear` handler must be of the :c:type:`inquiry` type, or ``NULL`` +if the object is immutable. +.. c:type:: int (*inquiry)(PyObject *self) + + Drop references that may have created reference cycles. Immutable objects + do not have to define this method since they can never directly create + reference cycles. Note that the object must still be valid after calling + this method (don't just call :c:func:`Py_DECREF` on a reference). The + collector will call this method if it detects that this object is involved + in a reference cycle. + + +.. _gc-traversal: + +Traversal +--------- + +The :c:member:`~PyTypeObject.tp_traverse` handler must have the following type: + .. c:type:: int (*traverseproc)(PyObject *self, visitproc visit, void *arg) - Traversal function for a container object. Implementations must call the + Traversal function for a garbage-collected object, used by the garbage + collector to detect reference cycles. + Implementations must call the *visit* function for each object directly contained by *self*, with the parameters to *visit* being the contained object and the *arg* value passed to the handler. The *visit* function must not be called with a ``NULL`` - object argument. If *visit* returns a non-zero value that value should be + object argument. If *visit* returns a non-zero value, that value should be returned immediately. -To simplify writing :c:member:`~PyTypeObject.tp_traverse` handlers, a :c:func:`Py_VISIT` macro is -provided. In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` implementation -must name its arguments exactly *visit* and *arg*: - - -.. c:macro:: Py_VISIT(o) - - If the :c:expr:`PyObject *` *o* is not ``NULL``, call the *visit* callback, with arguments *o* - and *arg*. If *visit* returns a non-zero value, then return it. - Using this macro, :c:member:`~PyTypeObject.tp_traverse` handlers - look like:: + A typical :c:member:`!tp_traverse` function calls the :c:func:`Py_VISIT` + convenience macro on each of the instance's members that are Python + objects that the instance owns. + For example, this is a (slightly outdated) traversal function for + the :py:class:`threading.local` class:: static int - my_traverse(Noddy *self, visitproc visit, void *arg) + local_traverse(PyObject *op, visitproc visit, void *arg) { - Py_VISIT(self->foo); - Py_VISIT(self->bar); + localobject *self = (localobject *) op; + Py_VISIT(Py_TYPE(self)); + Py_VISIT(self->args); + Py_VISIT(self->kw); + Py_VISIT(self->dict); return 0; } -The :c:member:`~PyTypeObject.tp_clear` handler must be of the :c:type:`inquiry` type, or ``NULL`` -if the object is immutable. + .. note:: + :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to + :c:func:`!local_traverse` to have these specific names; don't name them just + anything. + Instances of :ref:`heap-allocated types ` hold a reference to + their type. Their traversal function must therefore visit the type:: -.. c:type:: int (*inquiry)(PyObject *self) + Py_VISIT(Py_TYPE(self)); - Drop references that may have created reference cycles. Immutable objects - do not have to define this method since they can never directly create - reference cycles. Note that the object must still be valid after calling - this method (don't just call :c:func:`Py_DECREF` on a reference). The - collector will call this method if it detects that this object is involved - in a reference cycle. + Alternately, the type may delegate this responsibility by + calling ``tp_traverse`` of a heap-allocated superclass (or another + heap-allocated type, if applicable). + If they do not, the type object may not be garbage-collected. + + If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the + :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call + :c:func:`PyObject_VisitManagedDict` like this:: + + int err = PyObject_VisitManagedDict((PyObject*)self, visit, arg); + if (err) { + return err; + } + + Only the members that the instance *owns* (by having + :term:`strong references ` to them) must be + visited. For instance, if an object supports weak references via the + :c:member:`~PyTypeObject.tp_weaklist` slot, the pointer supporting + the linked list (what *tp_weaklist* points to) must **not** be + visited as the instance does not directly own the weak references to itself. + + The traversal function has a limitation: + + .. warning:: + + The traversal function must not have any side effects. Implementations + may not modify the reference counts of any Python objects nor create or + destroy any Python objects, directly or indirectly. + + This means that *most* Python C API functions may not be used, since + they can raise a new exception, return a new reference to a result object, + have internal logic that uses side effects. + Also, unless documented otherwise, functions that happen to not have side + effects may start having them in future versions, without warning. + + For a list of safe functions, see a + :ref:`separate section ` below. + + .. note:: + + The :c:func:`Py_VISIT` call may be skipped for those members that provably + cannot participate in reference cycles. + In the ``local_traverse`` example above, there is also a ``self->key`` + member, but it can only be ``NULL`` or a Python string and therefore + cannot be part of a reference cycle. + + On the other hand, even if you know a member can never be part of a cycle, + as a debugging aid you may want to visit it anyway just so the :mod:`gc` + module's :func:`~gc.get_referents` function will include it. + + .. note:: + + The :c:member:`~PyTypeObject.tp_traverse` function can be called from any + thread. + + .. impl-detail:: + + Garbage collection is a "stop-the-world" operation: + even in :term:`free threading` builds, only one thread state is + :term:`attached ` when :c:member:`!tp_traverse` + handlers run. + + .. versionchanged:: 3.9 + + Heap-allocated types are expected to visit ``Py_TYPE(self)`` in + ``tp_traverse``. In earlier versions of Python, due to + `bug 40217 `_, doing this + may lead to crashes in subclasses. + +To simplify writing :c:member:`~PyTypeObject.tp_traverse` handlers, +a :c:func:`Py_VISIT` macro is provided. +In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` +implementation must name its arguments exactly *visit* and *arg*: + +.. c:macro:: Py_VISIT(o) + + If the :c:expr:`PyObject *` *o* is not ``NULL``, call the *visit* + callback, with arguments *o* and *arg*. + If *visit* returns a non-zero value, then return it. + + This corresponds roughly to:: + + #define Py_VISIT(o) \ + if (op) { \ + int visit_result = visit(o, arg); \ + if (visit_result != 0) { \ + return visit_result; \ + } \ + } + + +Traversal-safe functions +^^^^^^^^^^^^^^^^^^^^^^^^ + +The following functions and macros are safe to use in a +:c:member:`~PyTypeObject.tp_traverse` handler: + +* the *visit* function passed to ``tp_traverse`` +* :c:func:`Py_VISIT` +* :c:func:`Py_SIZE` +* :c:func:`Py_TYPE`: if called from a :c:member:`!tp_traverse` handler, + :c:func:`!Py_TYPE`'s result will be valid for the duration of the handler call +* :c:func:`PyObject_VisitManagedDict` +* :c:func:`PyObject_TypeCheck`, :c:func:`PyType_IsSubtype`, + :c:func:`PyType_HasFeature` +* :samp:`Py{}_Check` and :samp:`Py{}_CheckExact` -- for example, + :c:func:`PyTuple_Check` +* :ref:`duringgc-functions` + +.. _duringgc-functions: + +"DuringGC" functions +^^^^^^^^^^^^^^^^^^^^ + +The following functions should *only* be used in a +:c:member:`~PyTypeObject.tp_traverse` handler; calling them in other +contexts may have unintended consequences. + +These functions act like their counterparts without the ``_DuringGC`` suffix, +but they are guaranteed to not have side effects, they do not set an exception +on failure, and they return/set :term:`borrowed references ` +as detailed in the individual documentation. + +Note that these functions may fail (return ``NULL`` or ``-1``), +but as they do not set an exception, no error information is available. +In some cases, failure is not distinguishable from a successful ``NULL`` result. + +.. c:function:: void *PyObject_GetTypeData_DuringGC(PyObject *o, PyTypeObject *cls) + void *PyObject_GetItemData_DuringGC(PyObject *o) + void *PyType_GetModuleState_DuringGC(PyTypeObject *type) + void *PyModule_GetState_DuringGC(PyObject *module) + int PyModule_GetToken_DuringGC(PyObject *module, void** result) + + See :ref:`duringgc-functions` for common information. + + .. versionadded:: 3.15 + + .. seealso:: + + :c:func:`PyObject_GetTypeData`, + :c:func:`PyObject_GetItemData`, + :c:func:`PyType_GetModuleState`, + :c:func:`PyModule_GetState`, + :c:func:`PyModule_GetToken`, + :c:func:`PyType_GetBaseByToken` + +.. c:function:: int PyType_GetBaseByToken_DuringGC(PyTypeObject *type, void *tp_token, PyTypeObject **result) + + See :ref:`duringgc-functions` for common information. + + Sets *\*result* to a :term:`borrowed reference` rather than a strong one. + The reference is valid for the duration + of the :c:member:`!tp_traverse` handler call. + + .. versionadded:: 3.15 + + .. seealso:: :c:func:`PyType_GetBaseByToken` + +.. c:function:: PyObject* PyType_GetModule_DuringGC(PyTypeObject *type) + PyObject* PyType_GetModuleByToken_DuringGC(PyTypeObject *type, const void *mod_token) + + See :ref:`duringgc-functions` for common information. + + These functions return a :term:`borrowed reference`, which is + valid for the duration of the :c:member:`!tp_traverse` handler call. + + .. versionadded:: 3.15 + + .. seealso:: + + :c:func:`PyType_GetModule`, + :c:func:`PyType_GetModuleByToken` Controlling the Garbage Collector State diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst index 0eb5922f6da75f2..7713ba2ee4f8047 100644 --- a/Doc/c-api/gen.rst +++ b/Doc/c-api/gen.rst @@ -38,9 +38,79 @@ than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`. A reference to *frame* is stolen by this function. The argument must not be ``NULL``. + .. deprecated-removed:: 3.16 3.18 + + This function has not been used since 3.10. + It is also impossible to construct a proper *frame* + object to call this function. + .. c:function:: PyObject* PyGen_NewWithQualName(PyFrameObject *frame, PyObject *name, PyObject *qualname) Create and return a new generator object based on the *frame* object, with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. A reference to *frame* is stolen by this function. The *frame* argument must not be ``NULL``. + + .. deprecated-removed:: 3.16 3.18 + + This function has not been used since 3.10. + It is also impossible to construct a proper *frame* + object to call this function. + + +.. c:function:: PyCodeObject* PyGen_GetCode(PyGenObject *gen) + + Return a new :term:`strong reference` to the code object wrapped by *gen*. + This function always succeeds. + + +Asynchronous Generator Objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. seealso:: + :pep:`525` + +.. c:var:: PyTypeObject PyAsyncGen_Type + + The type object corresponding to asynchronous generator objects. This is + available as :class:`types.AsyncGeneratorType` in the Python layer. + + .. versionadded:: 3.6 + +.. c:function:: PyObject *PyAsyncGen_New(PyFrameObject *frame, PyObject *name, PyObject *qualname) + + Create a new asynchronous generator wrapping *frame*, with ``__name__`` and + ``__qualname__`` set to *name* and *qualname*. *frame* is stolen by this + function and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to the + new asynchronous generator. On failure, this function returns ``NULL`` + with an exception set. + + .. versionadded:: 3.6 + + .. deprecated-removed:: 3.16 3.18 + + This function has not been used since 3.10. + It is also impossible to construct a proper *frame* + object to call this function. + +.. c:function:: int PyAsyncGen_CheckExact(PyObject *op) + + Return true if *op* is an asynchronous generator object, false otherwise. + This function always succeeds. + + .. versionadded:: 3.6 + + +Deprecated API +^^^^^^^^^^^^^^ + +.. c:macro:: PyAsyncGenASend_CheckExact(op) + + This is an API that was included in Python's C API + by mistake. + + It is solely here for completeness; do not use this API. + + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst index 00f8cb887dc7ebf..1ad712b0ce4f2b1 100644 --- a/Doc/c-api/hash.rst +++ b/Doc/c-api/hash.rst @@ -11,47 +11,103 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. versionadded:: 3.2 + .. c:type:: Py_uhash_t Hash value type: unsigned integer. .. versionadded:: 3.2 + +.. c:macro:: Py_HASH_ALGORITHM + + A numerical value indicating the algorithm for hashing of :class:`str`, + :class:`bytes`, and :class:`memoryview`. + + The algorithm name is exposed by :data:`sys.hash_info.algorithm`. + + .. versionadded:: 3.4 + + +.. c:macro:: Py_HASH_FNV + Py_HASH_SIPHASH24 + Py_HASH_SIPHASH13 + + Numerical values to compare to :c:macro:`Py_HASH_ALGORITHM` to determine + which algorithm is used for hashing. The hash algorithm can be configured + via the configure :option:`--with-hash-algorithm` option. + + .. versionadded:: 3.4 + Add :c:macro:`!Py_HASH_FNV` and :c:macro:`!Py_HASH_SIPHASH24`. + + .. versionadded:: 3.11 + Add :c:macro:`!Py_HASH_SIPHASH13`. + + +.. c:macro:: Py_HASH_CUTOFF + + Buffers of length in range ``[1, Py_HASH_CUTOFF)`` are hashed using DJBX33A + instead of the algorithm described by :c:macro:`Py_HASH_ALGORITHM`. + + - A :c:macro:`!Py_HASH_CUTOFF` of 0 disables the optimization. + - :c:macro:`!Py_HASH_CUTOFF` must be non-negative and less or equal than 7. + + 32-bit platforms should use a cutoff smaller than 64-bit platforms because + it is easier to create colliding strings. A cutoff of 7 on 64-bit platforms + and 5 on 32-bit platforms should provide a decent safety margin. + + This corresponds to the :data:`sys.hash_info.cutoff` constant. + + .. versionadded:: 3.4 + + .. c:macro:: PyHASH_MODULUS - The `Mersenne prime `_ ``P = 2**n -1``, used for numeric hash scheme. + The `Mersenne prime `_ ``P = 2**n -1``, + used for numeric hash scheme. + + This corresponds to the :data:`sys.hash_info.modulus` constant. .. versionadded:: 3.13 + .. c:macro:: PyHASH_BITS The exponent ``n`` of ``P`` in :c:macro:`PyHASH_MODULUS`. .. versionadded:: 3.13 + .. c:macro:: PyHASH_MULTIPLIER Prime multiplier used in string and various other hashes. .. versionadded:: 3.13 + .. c:macro:: PyHASH_INF The hash value returned for a positive infinity. + This corresponds to the :data:`sys.hash_info.inf` constant. + .. versionadded:: 3.13 + .. c:macro:: PyHASH_IMAG The multiplier used for the imaginary part of a complex number. + This corresponds to the :data:`sys.hash_info.imag` constant. + .. versionadded:: 3.13 + .. c:type:: PyHash_FuncDef Hash function definition used by :c:func:`PyHash_GetFuncDef`. - .. c::member:: Py_hash_t (*const hash)(const void *, Py_ssize_t) + .. c:member:: Py_hash_t (*const hash)(const void *, Py_ssize_t) Hash function. @@ -59,14 +115,20 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. Hash function name (UTF-8 encoded string). + This corresponds to the :data:`sys.hash_info.algorithm` constant. + .. c:member:: const int hash_bits Internal size of the hash value in bits. + This corresponds to the :data:`sys.hash_info.hash_bits` constant. + .. c:member:: const int seed_bits Size of seed input in bits. + This corresponds to the :data:`sys.hash_info.seed_bits` constant. + .. versionadded:: 3.4 diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 8eabc0406b11ce5..ec9462931d56c2c 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -129,8 +129,7 @@ Importing Modules of :class:`~importlib.machinery.SourceFileLoader` otherwise. The module's :attr:`~module.__file__` attribute will be set to the code - object's :attr:`~codeobject.co_filename`. If applicable, - :attr:`~module.__cached__` will also be set. + object's :attr:`~codeobject.co_filename`. This function will reload the module if it was already imported. See :c:func:`PyImport_ReloadModule` for the intended way to reload a module. @@ -142,10 +141,13 @@ Importing Modules :c:func:`PyImport_ExecCodeModuleWithPathnames`. .. versionchanged:: 3.12 - The setting of :attr:`~module.__cached__` and :attr:`~module.__loader__` + The setting of ``__cached__`` and :attr:`~module.__loader__` is deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. + .. versionchanged:: 3.15 + ``__cached__`` is no longer set. + .. c:function:: PyObject* PyImport_ExecCodeModuleEx(const char *name, PyObject *co, const char *pathname) @@ -157,16 +159,19 @@ Importing Modules .. c:function:: PyObject* PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, PyObject *cpathname) - Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`~module.__cached__` - attribute of the module object is set to *cpathname* if it is - non-``NULL``. Of the three functions, this is the preferred one to use. + Like :c:func:`PyImport_ExecCodeModuleEx`, but the path to any compiled file + via *cpathname* is used appropriately when non-``NULL``. Of the three + functions, this is the preferred one to use. .. versionadded:: 3.3 .. versionchanged:: 3.12 - Setting :attr:`~module.__cached__` is deprecated. See + Setting ``__cached__`` is deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. + .. versionchanged:: 3.15 + ``__cached__`` no longer set. + .. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(const char *name, PyObject *co, const char *pathname, const char *cpathname) @@ -314,6 +319,13 @@ Importing Modules initialization. +.. c:var:: struct _inittab *PyImport_Inittab + + The table of built-in modules used by Python initialization. Do not use this directly; + use :c:func:`PyImport_AppendInittab` and :c:func:`PyImport_ExtendInittab` + instead. + + .. c:function:: PyObject* PyImport_ImportModuleAttr(PyObject *mod_name, PyObject *attr_name) Import the module *mod_name* and get its attribute *attr_name*. @@ -333,3 +345,73 @@ Importing Modules strings instead of Python :class:`str` objects. .. versionadded:: 3.14 + +.. c:function:: PyImport_LazyImportsMode PyImport_GetLazyImportsMode() + + Gets the current lazy imports mode. + + .. versionadded:: 3.15 + +.. c:function:: PyObject* PyImport_GetLazyImportsFilter() + + Return a :term:`strong reference` to the current lazy imports filter, + or ``NULL`` if none exists. This function always succeeds. + + .. versionadded:: 3.15 + +.. c:function:: int PyImport_SetLazyImportsMode(PyImport_LazyImportsMode mode) + + Similar to :c:func:`PyImport_ImportModuleAttr`, but names are UTF-8 encoded + strings instead of Python :class:`str` objects. + + This function always returns ``0``. + + .. versionadded:: 3.15 + +.. c:function:: int PyImport_SetLazyImportsFilter(PyObject *filter) + + Sets the current lazy imports filter. The *filter* should be a callable that + will receive ``(importing_module_name, imported_module_name, [fromlist])`` + when an import can potentially be lazy. The ``imported_module_name`` value + is the resolved module name, so ``lazy from .spam import eggs`` passes + ``package.spam``. The callable must return ``True`` if the import should be + lazy and ``False`` otherwise. + + Return ``0`` on success and ``-1`` with an exception set otherwise. + + .. versionadded:: 3.15 + +.. c:type:: PyImport_LazyImportsMode + + Enumeration of possible lazy import modes. + + .. c:enumerator:: PyImport_LAZY_NORMAL + + Respect the ``lazy`` keyword in source code. This is the default mode. + + .. c:enumerator:: PyImport_LAZY_ALL + + Make all imports lazy by default. + + .. versionadded:: 3.15 + +.. c:function:: PyObject* PyImport_CreateModuleFromInitfunc(PyObject *spec, PyObject* (*initfunc)(void)) + + This function is a building block that enables embedders to implement + the :py:meth:`~importlib.abc.Loader.create_module` step of custom + static extension importers (e.g. of statically-linked extensions). + + *spec* must be a :class:`~importlib.machinery.ModuleSpec` object. + + *initfunc* must be an :ref:`initialization function `, + the same as for :c:func:`PyImport_AppendInittab`. + + On success, create and return a module object. + This module will not be initialized; call :c:func:`PyModule_Exec` + to initialize it. + (Custom importers should do this in their + :py:meth:`~importlib.abc.Loader.exec_module` method.) + + On error, return NULL with an exception set. + + .. versionadded:: 3.15 diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index e9df2a304d975b4..051f6fd765e8506 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -1,7 +1,7 @@ .. _c-api-index: ################################## - Python/C API Reference Manual + Python/C API reference manual ################################## This manual documents the API used by C and C++ programmers who want to write @@ -18,10 +18,16 @@ document the API functions in detail. refcounting.rst exceptions.rst extension-modules.rst + slots.rst utilities.rst abstract.rst concrete.rst - init.rst + interp-lifecycle.rst + threads.rst + synchronization.rst + tls.rst + subinterpreters.rst + profiling.rst init_config.rst memory.rst objimpl.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index e3ad4f4cdc52cc2..e56c67f95348c78 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1,2434 +1,13 @@ -.. highlight:: c +:orphan: +Initialization, finalization, and threads +========================================= -.. _initialization: +This page has been split up into the following: -***************************************** -Initialization, Finalization, and Threads -***************************************** - -See :ref:`Python Initialization Configuration ` for details -on how to configure the interpreter prior to initialization. - -.. _pre-init-safe: - -Before Python Initialization -============================ - -In an application embedding Python, the :c:func:`Py_Initialize` function must -be called before using any other Python/C API functions; with the exception of -a few functions and the :ref:`global configuration variables -`. - -The following functions can be safely called before Python is initialized: - -* Functions that initialize the interpreter: - - * :c:func:`Py_Initialize` - * :c:func:`Py_InitializeEx` - * :c:func:`Py_InitializeFromConfig` - * :c:func:`Py_BytesMain` - * :c:func:`Py_Main` - * the runtime pre-initialization functions covered in :ref:`init-config` - -* Configuration functions: - - * :c:func:`PyImport_AppendInittab` - * :c:func:`PyImport_ExtendInittab` - * :c:func:`!PyInitFrozenExtensions` - * :c:func:`PyMem_SetAllocator` - * :c:func:`PyMem_SetupDebugHooks` - * :c:func:`PyObject_SetArenaAllocator` - * :c:func:`Py_SetProgramName` - * :c:func:`Py_SetPythonHome` - * :c:func:`PySys_ResetWarnOptions` - * the configuration functions covered in :ref:`init-config` - -* Informative functions: - - * :c:func:`Py_IsInitialized` - * :c:func:`PyMem_GetAllocator` - * :c:func:`PyObject_GetArenaAllocator` - * :c:func:`Py_GetBuildInfo` - * :c:func:`Py_GetCompiler` - * :c:func:`Py_GetCopyright` - * :c:func:`Py_GetPlatform` - * :c:func:`Py_GetVersion` - * :c:func:`Py_IsInitialized` - -* Utilities: - - * :c:func:`Py_DecodeLocale` - * the status reporting and utility functions covered in :ref:`init-config` - -* Memory allocators: - - * :c:func:`PyMem_RawMalloc` - * :c:func:`PyMem_RawRealloc` - * :c:func:`PyMem_RawCalloc` - * :c:func:`PyMem_RawFree` - -* Synchronization: - - * :c:func:`PyMutex_Lock` - * :c:func:`PyMutex_Unlock` - -.. note:: - - Despite their apparent similarity to some of the functions listed above, - the following functions **should not be called** before the interpreter has - been initialized: :c:func:`Py_EncodeLocale`, :c:func:`PyEval_InitThreads`, and - :c:func:`Py_RunMain`. - - -.. _global-conf-vars: - -Global configuration variables -============================== - -Python has variables for the global configuration to control different features -and options. By default, these flags are controlled by :ref:`command line -options `. - -When a flag is set by an option, the value of the flag is the number of times -that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` -to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. - -.. c:var:: int Py_BytesWarningFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.bytes_warning` should be used instead, see :ref:`Python - Initialization Configuration `. - - Issue a warning when comparing :class:`bytes` or :class:`bytearray` with - :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater - or equal to ``2``. - - Set by the :option:`-b` option. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_DebugFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.parser_debug` should be used instead, see :ref:`Python - Initialization Configuration `. - - Turn on parser debugging output (for expert only, depending on compilation - options). - - Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_DontWriteBytecodeFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.write_bytecode` should be used instead, see :ref:`Python - Initialization Configuration `. - - If set to non-zero, Python won't try to write ``.pyc`` files on the - import of source modules. - - Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` - environment variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_FrozenFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.pathconfig_warnings` should be used instead, see - :ref:`Python Initialization Configuration `. - - Private flag used by ``_freeze_module`` and ``frozenmain`` programs. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_HashRandomizationFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.hash_seed` and :c:member:`PyConfig.use_hash_seed` should - be used instead, see :ref:`Python Initialization Configuration - `. - - Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to - a non-empty string. - - If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment - variable to initialize the secret hash seed. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_IgnoreEnvironmentFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.use_environment` should be used instead, see - :ref:`Python Initialization Configuration `. - - Ignore all :envvar:`!PYTHON*` environment variables, e.g. - :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. - - Set by the :option:`-E` and :option:`-I` options. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_InspectFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.inspect` should be used instead, see - :ref:`Python Initialization Configuration `. - - When a script is passed as first argument or the :option:`-c` option is used, - enter interactive mode after executing the script or the command, even when - :data:`sys.stdin` does not appear to be a terminal. - - Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_InteractiveFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.interactive` should be used instead, see - :ref:`Python Initialization Configuration `. - - Set by the :option:`-i` option. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_IsolatedFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.isolated` should be used instead, see - :ref:`Python Initialization Configuration `. - - Run Python in isolated mode. In isolated mode :data:`sys.path` contains - neither the script's directory nor the user's site-packages directory. - - Set by the :option:`-I` option. - - .. versionadded:: 3.4 - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_LegacyWindowsFSEncodingFlag - - This API is kept for backward compatibility: setting - :c:member:`PyPreConfig.legacy_windows_fs_encoding` should be used instead, see - :ref:`Python Initialization Configuration `. - - If the flag is non-zero, use the ``mbcs`` encoding with ``replace`` error - handler, instead of the UTF-8 encoding with ``surrogatepass`` error handler, - for the :term:`filesystem encoding and error handler`. - - Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment - variable is set to a non-empty string. - - See :pep:`529` for more details. - - .. availability:: Windows. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_LegacyWindowsStdioFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.legacy_windows_stdio` should be used instead, see - :ref:`Python Initialization Configuration `. - - If the flag is non-zero, use :class:`io.FileIO` instead of - :class:`!io._WindowsConsoleIO` for :mod:`sys` standard streams. - - Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment - variable is set to a non-empty string. - - See :pep:`528` for more details. - - .. availability:: Windows. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_NoSiteFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.site_import` should be used instead, see - :ref:`Python Initialization Configuration `. - - Disable the import of the module :mod:`site` and the site-dependent - manipulations of :data:`sys.path` that it entails. Also disable these - manipulations if :mod:`site` is explicitly imported later (call - :func:`site.main` if you want them to be triggered). - - Set by the :option:`-S` option. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_NoUserSiteDirectory - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.user_site_directory` should be used instead, see - :ref:`Python Initialization Configuration `. - - Don't add the :data:`user site-packages directory ` to - :data:`sys.path`. - - Set by the :option:`-s` and :option:`-I` options, and the - :envvar:`PYTHONNOUSERSITE` environment variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_OptimizeFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.optimization_level` should be used instead, see - :ref:`Python Initialization Configuration `. - - Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_QuietFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.quiet` should be used instead, see :ref:`Python - Initialization Configuration `. - - Don't display the copyright and version messages even in interactive mode. - - Set by the :option:`-q` option. - - .. versionadded:: 3.2 - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_UnbufferedStdioFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.buffered_stdio` should be used instead, see :ref:`Python - Initialization Configuration `. - - Force the stdout and stderr streams to be unbuffered. - - Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` - environment variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_VerboseFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.verbose` should be used instead, see :ref:`Python - Initialization Configuration `. - - Print a message each time a module is initialized, showing the place - (filename or built-in module) from which it is loaded. If greater or equal - to ``2``, print a message for each file that is checked for when - searching for a module. Also provides information on module cleanup at exit. - - Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - - -Initializing and finalizing the interpreter -=========================================== - - -.. c:function:: void Py_Initialize() - - .. index:: - single: PyEval_InitThreads() - single: modules (in module sys) - single: path (in module sys) - pair: module; builtins - pair: module; __main__ - pair: module; sys - triple: module; search; path - single: Py_FinalizeEx (C function) - - Initialize the Python interpreter. In an application embedding Python, - this should be called before using any other Python/C API functions; see - :ref:`Before Python Initialization ` for the few exceptions. - - This initializes the table of loaded modules (``sys.modules``), and creates - the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. - It also initializes the module search path (``sys.path``). It does not set - ``sys.argv``; use the :ref:`Python Initialization Configuration ` - API for that. This is a no-op when called for a second time (without calling - :c:func:`Py_FinalizeEx` first). There is no return value; it is a fatal - error if the initialization fails. - - Use :c:func:`Py_InitializeFromConfig` to customize the - :ref:`Python Initialization Configuration `. - - .. note:: - On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, - which will also affect non-Python uses of the console using the C Runtime. - - -.. c:function:: void Py_InitializeEx(int initsigs) - - This function works like :c:func:`Py_Initialize` if *initsigs* is ``1``. If - *initsigs* is ``0``, it skips initialization registration of signal handlers, - which may be useful when CPython is embedded as part of a larger application. - - Use :c:func:`Py_InitializeFromConfig` to customize the - :ref:`Python Initialization Configuration `. - - -.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) - - Initialize Python from *config* configuration, as described in - :ref:`init-from-config`. - - See the :ref:`init-config` section for details on pre-initializing the - interpreter, populating the runtime configuration structure, and querying - the returned status structure. - - -.. c:function:: int Py_IsInitialized() - - Return true (nonzero) when the Python interpreter has been initialized, false - (zero) if not. After :c:func:`Py_FinalizeEx` is called, this returns false until - :c:func:`Py_Initialize` is called again. - - -.. c:function:: int Py_IsFinalizing() - - Return true (non-zero) if the main Python interpreter is - :term:`shutting down `. Return false (zero) otherwise. - - .. versionadded:: 3.13 - - -.. c:function:: int Py_FinalizeEx() - - Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of - Python/C API functions, and destroy all sub-interpreters (see - :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since - the last call to :c:func:`Py_Initialize`. This is a no-op when called for a second - time (without calling :c:func:`Py_Initialize` again first). - - Since this is the reverse of :c:func:`Py_Initialize`, it should be called - in the same thread with the same interpreter active. That means - the main thread and the main interpreter. - This should never be called while :c:func:`Py_RunMain` is running. - - Normally the return value is ``0``. - If there were errors during finalization (flushing buffered data), - ``-1`` is returned. - - Note that Python will do a best effort at freeing all memory allocated by the Python - interpreter. Therefore, any C-Extension should make sure to correctly clean up all - of the preveiously allocated PyObjects before using them in subsequent calls to - :c:func:`Py_Initialize`. Otherwise it could introduce vulnerabilities and incorrect - behavior. - - This function is provided for a number of reasons. An embedding application - might want to restart Python without having to restart the application itself. - An application that has loaded the Python interpreter from a dynamically - loadable library (or DLL) might want to free all memory allocated by Python - before unloading the DLL. During a hunt for memory leaks in an application a - developer might want to free all memory allocated by Python before exiting from - the application. - - **Bugs and caveats:** The destruction of modules and objects in modules is done - in random order; this may cause destructors (:meth:`~object.__del__` methods) to fail - when they depend on other objects (even functions) or modules. Dynamically - loaded extension modules loaded by Python are not unloaded. Small amounts of - memory allocated by the Python interpreter may not be freed (if you find a leak, - please report it). Memory tied up in circular references between objects is not - freed. Interned strings will all be deallocated regardless of their reference count. - Some memory allocated by extension modules may not be freed. Some extensions may not - work properly if their initialization routine is called more than once; this can - happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` - more than once. :c:func:`Py_FinalizeEx` must not be called recursively from - within itself. Therefore, it must not be called by any code that may be run - as part of the interpreter shutdown process, such as :py:mod:`atexit` - handlers, object finalizers, or any code that may be run while flushing the - stdout and stderr files. - - .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx - - .. versionadded:: 3.6 - - -.. c:function:: void Py_Finalize() - - This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that - disregards the return value. - - -.. c:function:: int Py_BytesMain(int argc, char **argv) - - Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings, - allowing the calling application to delegate the text decoding step to - the CPython runtime. - - .. versionadded:: 3.8 - - -.. c:function:: int Py_Main(int argc, wchar_t **argv) - - The main program for the standard interpreter, encapsulating a full - initialization/finalization cycle, as well as additional - behaviour to implement reading configurations settings from the environment - and command line, and then executing ``__main__`` in accordance with - :ref:`using-on-cmdline`. - - This is made available for programs which wish to support the full CPython - command line interface, rather than just embedding a Python runtime in a - larger application. - - The *argc* and *argv* parameters are similar to those which are passed to a - C program's :c:func:`main` function, except that the *argv* entries are first - converted to ``wchar_t`` using :c:func:`Py_DecodeLocale`. It is also - important to note that the argument list entries may be modified to point to - strings other than those passed in (however, the contents of the strings - pointed to by the argument list are not modified). - - The return value is ``2`` if the argument list does not represent a valid - Python command line, and otherwise the same as :c:func:`Py_RunMain`. - - In terms of the CPython runtime configuration APIs documented in the - :ref:`runtime configuration ` section (and without accounting - for error handling), ``Py_Main`` is approximately equivalent to:: - - PyConfig config; - PyConfig_InitPythonConfig(&config); - PyConfig_SetArgv(&config, argc, argv); - Py_InitializeFromConfig(&config); - PyConfig_Clear(&config); - - Py_RunMain(); - - In normal usage, an embedding application will call this function - *instead* of calling :c:func:`Py_Initialize`, :c:func:`Py_InitializeEx` or - :c:func:`Py_InitializeFromConfig` directly, and all settings will be applied - as described elsewhere in this documentation. If this function is instead - called *after* a preceding runtime initialization API call, then exactly - which environmental and command line configuration settings will be updated - is version dependent (as it depends on which settings correctly support - being modified after they have already been set once when the runtime was - first initialized). - - -.. c:function:: int Py_RunMain(void) - - Executes the main module in a fully configured CPython runtime. - - Executes the command (:c:member:`PyConfig.run_command`), the script - (:c:member:`PyConfig.run_filename`) or the module - (:c:member:`PyConfig.run_module`) specified on the command line or in the - configuration. If none of these values are set, runs the interactive Python - prompt (REPL) using the ``__main__`` module's global namespace. - - If :c:member:`PyConfig.inspect` is not set (the default), the return value - will be ``0`` if the interpreter exits normally (that is, without raising - an exception), the exit status of an unhandled :exc:`SystemExit`, or ``1`` - for any other unhandled exception. - - If :c:member:`PyConfig.inspect` is set (such as when the :option:`-i` option - is used), rather than returning when the interpreter exits, execution will - instead resume in an interactive Python prompt (REPL) using the ``__main__`` - module's global namespace. If the interpreter exited with an exception, it - is immediately raised in the REPL session. The function return value is - then determined by the way the *REPL session* terminates: ``0``, ``1``, or - the status of a :exc:`SystemExit`, as specified above. - - This function always finalizes the Python interpreter before it returns. - - See :ref:`Python Configuration ` for an example of a - customized Python that always runs in isolated mode using - :c:func:`Py_RunMain`. - -.. c:function:: int PyUnstable_AtExit(PyInterpreterState *interp, void (*func)(void *), void *data) - - Register an :mod:`atexit` callback for the target interpreter *interp*. - This is similar to :c:func:`Py_AtExit`, but takes an explicit interpreter and - data pointer for the callback. - - There must be an :term:`attached thread state` for *interp*. - - .. versionadded:: 3.13 - -Process-wide parameters -======================= - - -.. c:function:: void Py_SetProgramName(const wchar_t *name) - - .. index:: - single: Py_Initialize() - single: main() - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python - Initialization Configuration `. - - This function should be called before :c:func:`Py_Initialize` is called for - the first time, if it is called at all. It tells the interpreter the value - of the ``argv[0]`` argument to the :c:func:`main` function of the program - (converted to wide characters). - This is used by some other functions below to find - the Python run-time libraries relative to the interpreter executable. The - default value is ``'python'``. The argument should point to a - zero-terminated wide character string in static storage whose contents will not - change for the duration of the program's execution. No code in the Python - interpreter will change the contents of this storage. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - .. deprecated-removed:: 3.11 3.15 - - -.. c:function:: const char* Py_GetVersion() - - Return the version of this Python interpreter. This is a string that looks - something like :: - - "3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]" - - .. index:: single: version (in module sys) - - The first word (up to the first space character) is the current Python version; - the first characters are the major and minor version separated by a - period. The returned string points into static storage; the caller should not - modify its value. The value is available to Python code as :data:`sys.version`. - - See also the :c:var:`Py_Version` constant. - - -.. c:function:: const char* Py_GetPlatform() - - .. index:: single: platform (in module sys) - - Return the platform identifier for the current platform. On Unix, this is - formed from the "official" name of the operating system, converted to lower - case, followed by the major revision number; e.g., for Solaris 2.x, which is - also known as SunOS 5.x, the value is ``'sunos5'``. On macOS, it is - ``'darwin'``. On Windows, it is ``'win'``. The returned string points into - static storage; the caller should not modify its value. The value is available - to Python code as ``sys.platform``. - - -.. c:function:: const char* Py_GetCopyright() - - Return the official copyright string for the current Python version, for example - - ``'Copyright 1991-1995 Stichting Mathematisch Centrum, Amsterdam'`` - - .. index:: single: copyright (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as ``sys.copyright``. - - -.. c:function:: const char* Py_GetCompiler() - - Return an indication of the compiler used to build the current Python version, - in square brackets, for example:: - - "[GCC 2.7.2.2]" - - .. index:: single: version (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as part of the variable - ``sys.version``. - - -.. c:function:: const char* Py_GetBuildInfo() - - Return information about the sequence number and build date and time of the - current Python interpreter instance, for example :: - - "#67, Aug 1 1997, 22:34:28" - - .. index:: single: version (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as part of the variable - ``sys.version``. - - -.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) - - .. index:: - single: main() - single: Py_FatalError() - single: argv (in module sys) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.argv`, :c:member:`PyConfig.parse_argv` and - :c:member:`PyConfig.safe_path` should be used instead, see :ref:`Python - Initialization Configuration `. - - Set :data:`sys.argv` based on *argc* and *argv*. These parameters are - similar to those passed to the program's :c:func:`main` function with the - difference that the first entry should refer to the script file to be - executed rather than the executable hosting the Python interpreter. If there - isn't a script that will be run, the first entry in *argv* can be an empty - string. If this function fails to initialize :data:`sys.argv`, a fatal - condition is signalled using :c:func:`Py_FatalError`. - - If *updatepath* is zero, this is all the function does. If *updatepath* - is non-zero, the function also modifies :data:`sys.path` according to the - following algorithm: - - - If the name of an existing script is passed in ``argv[0]``, the absolute - path of the directory where the script is located is prepended to - :data:`sys.path`. - - Otherwise (that is, if *argc* is ``0`` or ``argv[0]`` doesn't point - to an existing file name), an empty string is prepended to - :data:`sys.path`, which is the same as prepending the current working - directory (``"."``). - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` - members of the :ref:`Python Initialization Configuration `. - - .. note:: - It is recommended that applications embedding the Python interpreter - for purposes other than executing a single script pass ``0`` as *updatepath*, - and update :data:`sys.path` themselves if desired. - See :cve:`2008-5983`. - - On versions before 3.1.3, you can achieve the same effect by manually - popping the first :data:`sys.path` element after having called - :c:func:`PySys_SetArgv`, for example using:: - - PyRun_SimpleString("import sys; sys.path.pop(0)\n"); - - .. versionadded:: 3.1.3 - - .. XXX impl. doesn't seem consistent in allowing ``0``/``NULL`` for the params; - check w/ Guido. - - .. deprecated-removed:: 3.11 3.15 - - -.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.argv` and :c:member:`PyConfig.parse_argv` should be used - instead, see :ref:`Python Initialization Configuration `. - - This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set - to ``1`` unless the :program:`python` interpreter was started with the - :option:`-I`. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` - members of the :ref:`Python Initialization Configuration `. - - .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. - - .. deprecated-removed:: 3.11 3.15 - - -.. c:function:: void Py_SetPythonHome(const wchar_t *home) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.home` should be used instead, see :ref:`Python - Initialization Configuration `. - - Set the default "home" directory, that is, the location of the standard - Python libraries. See :envvar:`PYTHONHOME` for the meaning of the - argument string. - - The argument should point to a zero-terminated character string in static - storage whose contents will not change for the duration of the program's - execution. No code in the Python interpreter will change the contents of - this storage. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - .. deprecated-removed:: 3.11 3.15 - - -.. _threads: - -Thread State and the Global Interpreter Lock -============================================ - -.. index:: - single: global interpreter lock - single: interpreter lock - single: lock, interpreter - -Unless on a :term:`free-threaded ` build of :term:`CPython`, -the Python interpreter is not fully thread-safe. In order to support -multi-threaded Python programs, there's a global lock, called the :term:`global -interpreter lock` or :term:`GIL`, that must be held by the current thread before -it can safely access Python objects. Without the lock, even the simplest -operations could cause problems in a multi-threaded program: for example, when -two threads simultaneously increment the reference count of the same object, the -reference count could end up being incremented only once instead of twice. - -.. index:: single: setswitchinterval (in module sys) - -Therefore, the rule exists that only the thread that has acquired the -:term:`GIL` may operate on Python objects or call Python/C API functions. -In order to emulate concurrency of execution, the interpreter regularly -tries to switch threads (see :func:`sys.setswitchinterval`). The lock is also -released around potentially blocking I/O operations like reading or writing -a file, so that other Python threads can run in the meantime. - -.. index:: - single: PyThreadState (C type) - -The Python interpreter keeps some thread-specific bookkeeping information -inside a data structure called :c:type:`PyThreadState`, known as a :term:`thread state`. -Each OS thread has a thread-local pointer to a :c:type:`PyThreadState`; a thread state -referenced by this pointer is considered to be :term:`attached `. - -A thread can only have one :term:`attached thread state` at a time. An attached -thread state is typically analogous with holding the :term:`GIL`, except on -:term:`free-threaded ` builds. On builds with the :term:`GIL` enabled, -:term:`attaching ` a thread state will block until the :term:`GIL` -can be acquired. However, even on builds with the :term:`GIL` disabled, it is still required -to have an attached thread state to call most of the C API. - -In general, there will always be an :term:`attached thread state` when using Python's C API. -Only in some specific cases (such as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block) will the -thread not have an attached thread state. If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns -``NULL``. - -Detaching the thread state from extension code ----------------------------------------------- - -Most extension code manipulating the :term:`thread state` has the following simple -structure:: - - Save the thread state in a local variable. - ... Do some blocking I/O operation ... - Restore the thread state from the local variable. - -This is so common that a pair of macros exists to simplify it:: - - Py_BEGIN_ALLOW_THREADS - ... Do some blocking I/O operation ... - Py_END_ALLOW_THREADS - -.. index:: - single: Py_BEGIN_ALLOW_THREADS (C macro) - single: Py_END_ALLOW_THREADS (C macro) - -The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a -hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the -block. - -The block above expands to the following code:: - - PyThreadState *_save; - - _save = PyEval_SaveThread(); - ... Do some blocking I/O operation ... - PyEval_RestoreThread(_save); - -.. index:: - single: PyEval_RestoreThread (C function) - single: PyEval_SaveThread (C function) - -Here is how these functions work: - -The :term:`attached thread state` holds the :term:`GIL` for the entire interpreter. When detaching -the :term:`attached thread state`, the :term:`GIL` is released, allowing other threads to attach -a thread state to their own thread, thus getting the :term:`GIL` and can start executing. -The pointer to the prior :term:`attached thread state` is stored as a local variable. -Upon reaching :c:macro:`Py_END_ALLOW_THREADS`, the thread state that was -previously :term:`attached ` is passed to :c:func:`PyEval_RestoreThread`. -This function will block until another releases its :term:`thread state `, -thus allowing the old :term:`thread state ` to get re-attached and the -C API can be called again. - -For :term:`free-threaded ` builds, the :term:`GIL` is normally -out of the question, but detaching the :term:`thread state ` is still required -for blocking I/O and long operations. The difference is that threads don't have to wait for the :term:`GIL` -to be released to attach their thread state, allowing true multi-core parallelism. - -.. note:: - Calling system I/O functions is the most common use case for detaching - the :term:`thread state `, but it can also be useful before calling - long-running computations which don't need access to Python objects, such - as compression or cryptographic functions operating over memory buffers. - For example, the standard :mod:`zlib` and :mod:`hashlib` modules detach the - :term:`thread state ` when compressing or hashing data. - - -.. _gilstate: - -Non-Python created threads --------------------------- - -When threads are created using the dedicated Python APIs (such as the -:mod:`threading` module), a thread state is automatically associated to them -and the code showed above is therefore correct. However, when threads are -created from C (for example by a third-party library with its own thread -management), they don't hold the :term:`GIL`, because they don't have an -:term:`attached thread state`. - -If you need to call Python code from these threads (often this will be part -of a callback API provided by the aforementioned third-party library), -you must first register these threads with the interpreter by -creating an :term:`attached thread state` before you can start using the Python/C -API. When you are done, you should detach the :term:`thread state `, and -finally free it. - -The :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` functions do -all of the above automatically. The typical idiom for calling into Python -from a C thread is:: - - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); - - /* Perform Python actions here. */ - result = CallSomeFunction(); - /* evaluate result or handle exception */ - - /* Release the thread. No Python API allowed beyond this point. */ - PyGILState_Release(gstate); - -Note that the ``PyGILState_*`` functions assume there is only one global -interpreter (created automatically by :c:func:`Py_Initialize`). Python -supports the creation of additional interpreters (using -:c:func:`Py_NewInterpreter`), but mixing multiple interpreters and the -``PyGILState_*`` API is unsupported. This is because :c:func:`PyGILState_Ensure` -and similar functions default to :term:`attaching ` a -:term:`thread state` for the main interpreter, meaning that the thread can't safely -interact with the calling subinterpreter. - -Supporting subinterpreters in non-Python threads ------------------------------------------------- - -If you would like to support subinterpreters with non-Python created threads, you -must use the ``PyThreadState_*`` API instead of the traditional ``PyGILState_*`` -API. - -In particular, you must store the interpreter state from the calling -function and pass it to :c:func:`PyThreadState_New`, which will ensure that -the :term:`thread state` is targeting the correct interpreter:: - - /* The return value of PyInterpreterState_Get() from the - function that created this thread. */ - PyInterpreterState *interp = ThreadData->interp; - PyThreadState *tstate = PyThreadState_New(interp); - PyThreadState_Swap(tstate); - - /* GIL of the subinterpreter is now held. - Perform Python actions here. */ - result = CallSomeFunction(); - /* evaluate result or handle exception */ - - /* Destroy the thread state. No Python API allowed beyond this point. */ - PyThreadState_Clear(tstate); - PyThreadState_DeleteCurrent(); - -.. _fork-and-threads: - -Cautions about fork() ---------------------- - -Another important thing to note about threads is their behaviour in the face -of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a -process forks only the thread that issued the fork will exist. This has a -concrete impact both on how locks must be handled and on all stored state -in CPython's runtime. - -The fact that only the "current" thread remains -means any locks held by other threads will never be released. Python solves -this for :func:`os.fork` by acquiring the locks it uses internally before -the fork, and releasing them afterwards. In addition, it resets any -:ref:`lock-objects` in the child. When extending or embedding Python, there -is no way to inform Python of additional (non-Python) locks that need to be -acquired before or reset after a fork. OS facilities such as -:c:func:`!pthread_atfork` would need to be used to accomplish the same thing. -Additionally, when extending or embedding Python, calling :c:func:`fork` -directly rather than through :func:`os.fork` (and returning to or calling -into Python) may result in a deadlock by one of Python's internal locks -being held by a thread that is defunct after the fork. -:c:func:`PyOS_AfterFork_Child` tries to reset the necessary locks, but is not -always able to. - -The fact that all other threads go away also means that CPython's -runtime state there must be cleaned up properly, which :func:`os.fork` -does. This means finalizing all other :c:type:`PyThreadState` objects -belonging to the current interpreter and all other -:c:type:`PyInterpreterState` objects. Due to this and the special -nature of the :ref:`"main" interpreter `, -:c:func:`fork` should only be called in that interpreter's "main" -thread, where the CPython global runtime was originally initialized. -The only exception is if :c:func:`exec` will be called immediately -after. - -.. _cautions-regarding-runtime-finalization: - -Cautions regarding runtime finalization ---------------------------------------- - -In the late stage of :term:`interpreter shutdown`, after attempting to wait for -non-daemon threads to exit (though this can be interrupted by -:class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime -is marked as *finalizing*: :c:func:`Py_IsFinalizing` and -:func:`sys.is_finalizing` return true. At this point, only the *finalization -thread* that initiated finalization (typically the main thread) is allowed to -acquire the :term:`GIL`. - -If any thread, other than the finalization thread, attempts to attach a :term:`thread state` -during finalization, either explicitly or -implicitly, the thread enters **a permanently blocked state** -where it remains until the program exits. In most cases this is harmless, but this can result -in deadlock if a later stage of finalization attempts to acquire a lock owned by the -blocked thread, or otherwise waits on the blocked thread. - -Gross? Yes. This prevents random crashes and/or unexpectedly skipped C++ -finalizations further up the call stack when such threads were forcibly exited -here in CPython 3.13 and earlier. The CPython runtime :term:`thread state` C APIs -have never had any error reporting or handling expectations at :term:`thread state` -attachment time that would've allowed for graceful exit from this situation. Changing that -would require new stable C APIs and rewriting the majority of C code in the -CPython ecosystem to use those with error handling. - - -High-level API --------------- - -These are the most commonly used types and functions when writing C extension -code, or when embedding the Python interpreter: - -.. c:type:: PyInterpreterState - - This data structure represents the state shared by a number of cooperating - threads. Threads belonging to the same interpreter share their module - administration and a few other internal items. There are no public members in - this structure. - - Threads belonging to different interpreters initially share nothing, except - process state like available memory, open file descriptors and such. The global - interpreter lock is also shared by all threads, regardless of to which - interpreter they belong. - - -.. c:type:: PyThreadState - - This data structure represents the state of a single thread. The only public - data member is: - - .. c:member:: PyInterpreterState *interp - - This thread's interpreter state. - - -.. c:function:: void PyEval_InitThreads() - - .. index:: - single: PyEval_AcquireThread() - single: PyEval_ReleaseThread() - single: PyEval_SaveThread() - single: PyEval_RestoreThread() - - Deprecated function which does nothing. - - In Python 3.6 and older, this function created the GIL if it didn't exist. - - .. versionchanged:: 3.9 - The function now does nothing. - - .. versionchanged:: 3.7 - This function is now called by :c:func:`Py_Initialize()`, so you don't - have to call it yourself anymore. - - .. versionchanged:: 3.2 - This function cannot be called before :c:func:`Py_Initialize()` anymore. - - .. deprecated:: 3.9 - - .. index:: pair: module; _thread - - -.. c:function:: PyThreadState* PyEval_SaveThread() - - Detach the :term:`attached thread state` and return it. - The thread will have no :term:`thread state` upon returning. - - -.. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) - - Set the :term:`attached thread state` to *tstate*. - The passed :term:`thread state` **should not** be :term:`attached `, - otherwise deadlock ensues. *tstate* will be attached upon returning. - - .. note:: - Calling this function from a thread when the runtime is finalizing will - hang the thread until the program exits, even if the thread was not - created by Python. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - -.. c:function:: PyThreadState* PyThreadState_Get() - - Return the :term:`attached thread state`. If the thread has no attached - thread state, (such as when inside of :c:macro:`Py_BEGIN_ALLOW_THREADS` - block), then this issues a fatal error (so that the caller needn't check - for ``NULL``). - - See also :c:func:`PyThreadState_GetUnchecked`. - -.. c:function:: PyThreadState* PyThreadState_GetUnchecked() - - Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a - fatal error if it is NULL. The caller is responsible to check if the result - is NULL. - - .. versionadded:: 3.13 - In Python 3.5 to 3.12, the function was private and known as - ``_PyThreadState_UncheckedGet()``. - - -.. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) - - Set the :term:`attached thread state` to *tstate*, and return the - :term:`thread state` that was attached prior to calling. - - This function is safe to call without an :term:`attached thread state`; it - will simply return ``NULL`` indicating that there was no prior thread state. - - .. seealso: - :c:func:`PyEval_ReleaseThread` - - .. note:: - Similar to :c:func:`PyGILState_Ensure`, this function will hang the - thread if the runtime is finalizing. - - -The following functions use thread-local storage, and are not compatible -with sub-interpreters: - -.. c:function:: PyGILState_STATE PyGILState_Ensure() - - Ensure that the current thread is ready to call the Python C API regardless - of the current state of Python, or of the :term:`attached thread state`. This may - be called as many times as desired by a thread as long as each call is - matched with a call to :c:func:`PyGILState_Release`. In general, other - thread-related APIs may be used between :c:func:`PyGILState_Ensure` and - :c:func:`PyGILState_Release` calls as long as the thread state is restored to - its previous state before the Release(). For example, normal usage of the - :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is - acceptable. - - The return value is an opaque "handle" to the :term:`attached thread state` when - :c:func:`PyGILState_Ensure` was called, and must be passed to - :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even - though recursive calls are allowed, these handles *cannot* be shared - each - unique call to :c:func:`PyGILState_Ensure` must save the handle for its call - to :c:func:`PyGILState_Release`. - - When the function returns, there will be an :term:`attached thread state` - and the thread will be able to call arbitrary Python code. Failure is a fatal error. - - .. warning:: - Calling this function when the runtime is finalizing is unsafe. Doing - so will either hang the thread until the program ends, or fully crash - the interpreter in rare cases. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - -.. c:function:: void PyGILState_Release(PyGILState_STATE) - - Release any resources previously acquired. After this call, Python's state will - be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call - (but generally this state will be unknown to the caller, hence the use of the - GILState API). - - Every call to :c:func:`PyGILState_Ensure` must be matched by a call to - :c:func:`PyGILState_Release` on the same thread. - -.. c:function:: PyThreadState* PyGILState_GetThisThreadState() - - Get the :term:`attached thread state` for this thread. May return ``NULL`` if no - GILState API has been used on the current thread. Note that the main thread - always has such a thread-state, even if no auto-thread-state call has been - made on the main thread. This is mainly a helper/diagnostic function. - - .. note:: - This function does not account for :term:`thread states ` created - by something other than :c:func:`PyGILState_Ensure` (such as :c:func:`PyThreadState_New`). - Prefer :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` - for most cases. - - .. seealso: :c:func:`PyThreadState_Get`` - -.. c:function:: int PyGILState_Check() - - Return ``1`` if the current thread is holding the :term:`GIL` and ``0`` otherwise. - This function can be called from any thread at any time. - Only if it has had its :term:`thread state ` initialized - via :c:func:`PyGILState_Ensure` will it return ``1``. - This is mainly a helper/diagnostic function. It can be useful - for example in callback contexts or memory allocation functions when - knowing that the :term:`GIL` is locked can allow the caller to perform sensitive - actions or otherwise behave differently. - - .. note:: - If the current Python process has ever created a subinterpreter, this - function will *always* return ``1``. Prefer :c:func:`PyThreadState_GetUnchecked` - for most cases. - - .. versionadded:: 3.4 - - -The following macros are normally used without a trailing semicolon; look for -example usage in the Python source distribution. - - -.. c:macro:: Py_BEGIN_ALLOW_THREADS - - This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. - Note that it contains an opening brace; it must be matched with a following - :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this - macro. - - -.. c:macro:: Py_END_ALLOW_THREADS - - This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains - a closing brace; it must be matched with an earlier - :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of - this macro. - - -.. c:macro:: Py_BLOCK_THREADS - - This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to - :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. - - -.. c:macro:: Py_UNBLOCK_THREADS - - This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to - :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable - declaration. - - -Low-level API -------------- - -All of the following functions must be called after :c:func:`Py_Initialize`. - -.. versionchanged:: 3.7 - :c:func:`Py_Initialize()` now initializes the :term:`GIL` - and sets an :term:`attached thread state`. - - -.. c:function:: PyInterpreterState* PyInterpreterState_New() - - Create a new interpreter state object. An :term:`attached thread state` is not needed, - but may optionally exist if it is necessary to serialize calls to this - function. - - .. audit-event:: cpython.PyInterpreterState_New "" c.PyInterpreterState_New - - -.. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp) - - Reset all information in an interpreter state object. There must be - an :term:`attached thread state` for the interpreter. - - .. audit-event:: cpython.PyInterpreterState_Clear "" c.PyInterpreterState_Clear - - -.. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp) - - Destroy an interpreter state object. There **should not** be an - :term:`attached thread state` for the target interpreter. The interpreter - state must have been reset with a previous call to :c:func:`PyInterpreterState_Clear`. - - -.. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) - - Create a new thread state object belonging to the given interpreter object. - An :term:`attached thread state` is not needed. - -.. c:function:: void PyThreadState_Clear(PyThreadState *tstate) - - Reset all information in a :term:`thread state` object. *tstate* - must be :term:`attached ` - - .. versionchanged:: 3.9 - This function now calls the :c:member:`PyThreadState.on_delete` callback. - Previously, that happened in :c:func:`PyThreadState_Delete`. - - .. versionchanged:: 3.13 - The :c:member:`PyThreadState.on_delete` callback was removed. - - -.. c:function:: void PyThreadState_Delete(PyThreadState *tstate) - - Destroy a :term:`thread state` object. *tstate* should not - be :term:`attached ` to any thread. - *tstate* must have been reset with a previous call to - :c:func:`PyThreadState_Clear`. - - -.. c:function:: void PyThreadState_DeleteCurrent(void) - - Detach the :term:`attached thread state` (which must have been reset - with a previous call to :c:func:`PyThreadState_Clear`) and then destroy it. - - No :term:`thread state` will be :term:`attached ` upon - returning. - -.. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) - - Get the current frame of the Python thread state *tstate*. - - Return a :term:`strong reference`. Return ``NULL`` if no frame is currently - executing. - - See also :c:func:`PyEval_GetFrame`. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: uint64_t PyThreadState_GetID(PyThreadState *tstate) - - Get the unique :term:`thread state` identifier of the Python thread state *tstate*. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: PyInterpreterState* PyThreadState_GetInterpreter(PyThreadState *tstate) - - Get the interpreter of the Python thread state *tstate*. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: void PyThreadState_EnterTracing(PyThreadState *tstate) - - Suspend tracing and profiling in the Python thread state *tstate*. - - Resume them using the :c:func:`PyThreadState_LeaveTracing` function. - - .. versionadded:: 3.11 - - -.. c:function:: void PyThreadState_LeaveTracing(PyThreadState *tstate) - - Resume tracing and profiling in the Python thread state *tstate* suspended - by the :c:func:`PyThreadState_EnterTracing` function. - - See also :c:func:`PyEval_SetTrace` and :c:func:`PyEval_SetProfile` - functions. - - .. versionadded:: 3.11 - - -.. c:function:: PyInterpreterState* PyInterpreterState_Get(void) - - Get the current interpreter. - - Issue a fatal error if there no :term:`attached thread state`. - It cannot return NULL. - - .. versionadded:: 3.9 - - -.. c:function:: int64_t PyInterpreterState_GetID(PyInterpreterState *interp) - - Return the interpreter's unique ID. If there was any error in doing - so then ``-1`` is returned and an error is set. - - The caller must have an :term:`attached thread state`. - - .. versionadded:: 3.7 - - -.. c:function:: PyObject* PyInterpreterState_GetDict(PyInterpreterState *interp) - - Return a dictionary in which interpreter-specific data may be stored. - If this function returns ``NULL`` then no exception has been raised and - the caller should assume no interpreter-specific dict is available. - - This is not a replacement for :c:func:`PyModule_GetState()`, which - extensions should use to store interpreter-specific state information. - - .. versionadded:: 3.8 - - -.. c:type:: PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) - - Type of a frame evaluation function. - - The *throwflag* parameter is used by the ``throw()`` method of generators: - if non-zero, handle the current exception. - - .. versionchanged:: 3.9 - The function now takes a *tstate* parameter. - - .. versionchanged:: 3.11 - The *frame* parameter changed from ``PyFrameObject*`` to ``_PyInterpreterFrame*``. - -.. c:function:: _PyFrameEvalFunction _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) - - Get the frame evaluation function. - - See the :pep:`523` "Adding a frame evaluation API to CPython". - - .. versionadded:: 3.9 - -.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) - - Set the frame evaluation function. - - See the :pep:`523` "Adding a frame evaluation API to CPython". - - .. versionadded:: 3.9 - - -.. c:function:: PyObject* PyThreadState_GetDict() - - Return a dictionary in which extensions can store thread-specific state - information. Each extension should use a unique key to use to store state in - the dictionary. It is okay to call this function when no :term:`thread state` - is :term:`attached `. If this function returns - ``NULL``, no exception has been raised and the caller should assume no - thread state is attached. - - -.. c:function:: int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) - - Asynchronously raise an exception in a thread. The *id* argument is the thread - id of the target thread; *exc* is the exception object to be raised. This - function does not steal any references to *exc*. To prevent naive misuse, you - must write your own C extension to call this. Must be called with an :term:`attached thread state`. - Returns the number of thread states modified; this is normally one, but will be - zero if the thread id isn't found. If *exc* is ``NULL``, the pending - exception (if any) for the thread is cleared. This raises no exceptions. - - .. versionchanged:: 3.7 - The type of the *id* parameter changed from :c:expr:`long` to - :c:expr:`unsigned long`. - -.. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) - - :term:`Attach ` *tstate* to the current thread, - which must not be ``NULL`` or already :term:`attached `. - - The calling thread must not already have an :term:`attached thread state`. - - .. note:: - Calling this function from a thread when the runtime is finalizing will - hang the thread until the program exits, even if the thread was not - created by Python. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.8 - Updated to be consistent with :c:func:`PyEval_RestoreThread`, - :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, - and terminate the current thread if called while the interpreter is finalizing. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - - :c:func:`PyEval_RestoreThread` is a higher-level function which is always - available (even when threads have not been initialized). - - -.. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) - - Detach the :term:`attached thread state`. - The *tstate* argument, which must not be ``NULL``, is only used to check - that it represents the :term:`attached thread state` --- if it isn't, a fatal error is - reported. - - :c:func:`PyEval_SaveThread` is a higher-level function which is always - available (even when threads have not been initialized). - - -.. _sub-interpreter-support: - -Sub-interpreter support -======================= - -While in most uses, you will only embed a single Python interpreter, there -are cases where you need to create several independent interpreters in the -same process and perhaps even in the same thread. Sub-interpreters allow -you to do that. - -The "main" interpreter is the first one created when the runtime initializes. -It is usually the only Python interpreter in a process. Unlike sub-interpreters, -the main interpreter has unique process-global responsibilities like signal -handling. It is also responsible for execution during runtime initialization and -is usually the active interpreter during runtime finalization. The -:c:func:`PyInterpreterState_Main` function returns a pointer to its state. - -You can switch between sub-interpreters using the :c:func:`PyThreadState_Swap` -function. You can create and destroy them using the following functions: - - -.. c:type:: PyInterpreterConfig - - Structure containing most parameters to configure a sub-interpreter. - Its values are used only in :c:func:`Py_NewInterpreterFromConfig` and - never modified by the runtime. - - .. versionadded:: 3.12 - - Structure fields: - - .. c:member:: int use_main_obmalloc - - If this is ``0`` then the sub-interpreter will use its own - "object" allocator state. - Otherwise it will use (share) the main interpreter's. - - If this is ``0`` then - :c:member:`~PyInterpreterConfig.check_multi_interp_extensions` - must be ``1`` (non-zero). - If this is ``1`` then :c:member:`~PyInterpreterConfig.gil` - must not be :c:macro:`PyInterpreterConfig_OWN_GIL`. - - .. c:member:: int allow_fork - - If this is ``0`` then the runtime will not support forking the - process in any thread where the sub-interpreter is currently active. - Otherwise fork is unrestricted. - - Note that the :mod:`subprocess` module still works - when fork is disallowed. - - .. c:member:: int allow_exec - - If this is ``0`` then the runtime will not support replacing the - current process via exec (e.g. :func:`os.execv`) in any thread - where the sub-interpreter is currently active. - Otherwise exec is unrestricted. - - Note that the :mod:`subprocess` module still works - when exec is disallowed. - - .. c:member:: int allow_threads - - If this is ``0`` then the sub-interpreter's :mod:`threading` module - won't create threads. - Otherwise threads are allowed. - - .. c:member:: int allow_daemon_threads - - If this is ``0`` then the sub-interpreter's :mod:`threading` module - won't create daemon threads. - Otherwise daemon threads are allowed (as long as - :c:member:`~PyInterpreterConfig.allow_threads` is non-zero). - - .. c:member:: int check_multi_interp_extensions - - If this is ``0`` then all extension modules may be imported, - including legacy (single-phase init) modules, - in any thread where the sub-interpreter is currently active. - Otherwise only multi-phase init extension modules - (see :pep:`489`) may be imported. - (Also see :c:macro:`Py_mod_multiple_interpreters`.) - - This must be ``1`` (non-zero) if - :c:member:`~PyInterpreterConfig.use_main_obmalloc` is ``0``. - - .. c:member:: int gil - - This determines the operation of the GIL for the sub-interpreter. - It may be one of the following: - - .. c:namespace:: NULL - - .. c:macro:: PyInterpreterConfig_DEFAULT_GIL - - Use the default selection (:c:macro:`PyInterpreterConfig_SHARED_GIL`). - - .. c:macro:: PyInterpreterConfig_SHARED_GIL - - Use (share) the main interpreter's GIL. - - .. c:macro:: PyInterpreterConfig_OWN_GIL - - Use the sub-interpreter's own GIL. - - If this is :c:macro:`PyInterpreterConfig_OWN_GIL` then - :c:member:`PyInterpreterConfig.use_main_obmalloc` must be ``0``. - - -.. c:function:: PyStatus Py_NewInterpreterFromConfig(PyThreadState **tstate_p, const PyInterpreterConfig *config) - - .. index:: - pair: module; builtins - pair: module; __main__ - pair: module; sys - single: stdout (in module sys) - single: stderr (in module sys) - single: stdin (in module sys) - - Create a new sub-interpreter. This is an (almost) totally separate environment - for the execution of Python code. In particular, the new interpreter has - separate, independent versions of all imported modules, including the - fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. The - table of loaded modules (``sys.modules``) and the module search path - (``sys.path``) are also separate. The new environment has no ``sys.argv`` - variable. It has new standard I/O stream file objects ``sys.stdin``, - ``sys.stdout`` and ``sys.stderr`` (however these refer to the same underlying - file descriptors). - - The given *config* controls the options with which the interpreter - is initialized. - - Upon success, *tstate_p* will be set to the first :term:`thread state` - created in the new sub-interpreter. This thread state is - :term:`attached `. - Note that no actual thread is created; see the discussion of thread states - below. If creation of the new interpreter is unsuccessful, - *tstate_p* is set to ``NULL``; - no exception is set since the exception state is stored in the - :term:`attached thread state`, which might not exist. - - Like all other Python/C API functions, an :term:`attached thread state` - must be present before calling this function, but it might be detached upon - returning. On success, the returned thread state will be :term:`attached `. - If the sub-interpreter is created with its own :term:`GIL` then the - :term:`attached thread state` of the calling interpreter will be detached. - When the function returns, the new interpreter's :term:`thread state` - will be :term:`attached ` to the current thread and - the previous interpreter's :term:`attached thread state` will remain detached. - - .. versionadded:: 3.12 - - Sub-interpreters are most effective when isolated from each other, - with certain functionality restricted:: - - PyInterpreterConfig config = { - .use_main_obmalloc = 0, - .allow_fork = 0, - .allow_exec = 0, - .allow_threads = 1, - .allow_daemon_threads = 0, - .check_multi_interp_extensions = 1, - .gil = PyInterpreterConfig_OWN_GIL, - }; - PyThreadState *tstate = NULL; - PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } - - Note that the config is used only briefly and does not get modified. - During initialization the config's values are converted into various - :c:type:`PyInterpreterState` values. A read-only copy of the config - may be stored internally on the :c:type:`PyInterpreterState`. - - .. index:: - single: Py_FinalizeEx (C function) - single: Py_Initialize (C function) - - Extension modules are shared between (sub-)interpreters as follows: - - * For modules using multi-phase initialization, - e.g. :c:func:`PyModule_FromDefAndSpec`, a separate module object is - created and initialized for each interpreter. - Only C-level static and global variables are shared between these - module objects. - - * For modules using single-phase initialization, - e.g. :c:func:`PyModule_Create`, the first time a particular extension - is imported, it is initialized normally, and a (shallow) copy of its - module's dictionary is squirreled away. - When the same extension is imported by another (sub-)interpreter, a new - module is initialized and filled with the contents of this copy; the - extension's ``init`` function is not called. - Objects in the module's dictionary thus end up shared across - (sub-)interpreters, which might cause unwanted behavior (see - `Bugs and caveats`_ below). - - Note that this is different from what happens when an extension is - imported after the interpreter has been completely re-initialized by - calling :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that - case, the extension's ``initmodule`` function *is* called again. - As with multi-phase initialization, this means that only C-level static - and global variables are shared between these modules. - - .. index:: single: close (in module os) - - -.. c:function:: PyThreadState* Py_NewInterpreter(void) - - .. index:: - pair: module; builtins - pair: module; __main__ - pair: module; sys - single: stdout (in module sys) - single: stderr (in module sys) - single: stdin (in module sys) - - Create a new sub-interpreter. This is essentially just a wrapper - around :c:func:`Py_NewInterpreterFromConfig` with a config that - preserves the existing behavior. The result is an unisolated - sub-interpreter that shares the main interpreter's GIL, allows - fork/exec, allows daemon threads, and allows single-phase init - modules. - - -.. c:function:: void Py_EndInterpreter(PyThreadState *tstate) - - .. index:: single: Py_FinalizeEx (C function) - - Destroy the (sub-)interpreter represented by the given :term:`thread state`. - The given thread state must be :term:`attached `. - When the call returns, there will be no :term:`attached thread state`. - All thread states associated with this interpreter are destroyed. - - :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that - haven't been explicitly destroyed at that point. - - -A Per-Interpreter GIL ---------------------- - -Using :c:func:`Py_NewInterpreterFromConfig` you can create -a sub-interpreter that is completely isolated from other interpreters, -including having its own GIL. The most important benefit of this -isolation is that such an interpreter can execute Python code without -being blocked by other interpreters or blocking any others. Thus a -single Python process can truly take advantage of multiple CPU cores -when running Python code. The isolation also encourages a different -approach to concurrency than that of just using threads. -(See :pep:`554`.) - -Using an isolated interpreter requires vigilance in preserving that -isolation. That especially means not sharing any objects or mutable -state without guarantees about thread-safety. Even objects that are -otherwise immutable (e.g. ``None``, ``(1, 5)``) can't normally be shared -because of the refcount. One simple but less-efficient approach around -this is to use a global lock around all use of some state (or object). -Alternately, effectively immutable objects (like integers or strings) -can be made safe in spite of their refcounts by making them :term:`immortal`. -In fact, this has been done for the builtin singletons, small integers, -and a number of other builtin objects. - -If you preserve isolation then you will have access to proper multi-core -computing without the complications that come with free-threading. -Failure to preserve isolation will expose you to the full consequences -of free-threading, including races and hard-to-debug crashes. - -Aside from that, one of the main challenges of using multiple isolated -interpreters is how to communicate between them safely (not break -isolation) and efficiently. The runtime and stdlib do not provide -any standard approach to this yet. A future stdlib module would help -mitigate the effort of preserving isolation and expose effective tools -for communicating (and sharing) data between interpreters. - -.. versionadded:: 3.12 - - -Bugs and caveats ----------------- - -Because sub-interpreters (and the main interpreter) are part of the same -process, the insulation between them isn't perfect --- for example, using -low-level file operations like :func:`os.close` they can -(accidentally or maliciously) affect each other's open files. Because of the -way extensions are shared between (sub-)interpreters, some extensions may not -work properly; this is especially likely when using single-phase initialization -or (static) global variables. -It is possible to insert objects created in one sub-interpreter into -a namespace of another (sub-)interpreter; this should be avoided if possible. - -Special care should be taken to avoid sharing user-defined functions, -methods, instances or classes between sub-interpreters, since import -operations executed by such objects may affect the wrong (sub-)interpreter's -dictionary of loaded modules. It is equally important to avoid sharing -objects from which the above are reachable. - -Also note that combining this functionality with ``PyGILState_*`` APIs -is delicate, because these APIs assume a bijection between Python thread states -and OS-level threads, an assumption broken by the presence of sub-interpreters. -It is highly recommended that you don't switch sub-interpreters between a pair -of matching :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` calls. -Furthermore, extensions (such as :mod:`ctypes`) using these APIs to allow calling -of Python code from non-Python created threads will probably be broken when using -sub-interpreters. - - -Asynchronous Notifications -========================== - -A mechanism is provided to make asynchronous notifications to the main -interpreter thread. These notifications take the form of a function -pointer and a void pointer argument. - - -.. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) - - Schedule a function to be called from the main interpreter thread. On - success, ``0`` is returned and *func* is queued for being called in the - main thread. On failure, ``-1`` is returned without setting any exception. - - When successfully queued, *func* will be *eventually* called from the - main interpreter thread with the argument *arg*. It will be called - asynchronously with respect to normally running Python code, but with - both these conditions met: - - * on a :term:`bytecode` boundary; - * with the main thread holding an :term:`attached thread state` - (*func* can therefore use the full C API). - - *func* must return ``0`` on success, or ``-1`` on failure with an exception - set. *func* won't be interrupted to perform another asynchronous - notification recursively, but it can still be interrupted to switch - threads if the :term:`thread state ` is detached. - - This function doesn't need an :term:`attached thread state`. However, to call this - function in a subinterpreter, the caller must have an :term:`attached thread state`. - Otherwise, the function *func* can be scheduled to be called from the wrong interpreter. - - .. warning:: - This is a low-level function, only useful for very special cases. - There is no guarantee that *func* will be called as quick as - possible. If the main thread is busy executing a system call, - *func* won't be called before the system call returns. This - function is generally **not** suitable for calling Python code from - arbitrary C threads. Instead, use the :ref:`PyGILState API`. - - .. versionadded:: 3.1 - - .. versionchanged:: 3.9 - If this function is called in a subinterpreter, the function *func* is - now scheduled to be called from the subinterpreter, rather than being - called from the main interpreter. Each subinterpreter now has its own - list of scheduled calls. - -.. _profiling: - -Profiling and Tracing -===================== - -.. sectionauthor:: Fred L. Drake, Jr. - - -The Python interpreter provides some low-level support for attaching profiling -and execution tracing facilities. These are used for profiling, debugging, and -coverage analysis tools. - -This C interface allows the profiling or tracing code to avoid the overhead of -calling through Python-level callable objects, making a direct C function call -instead. The essential attributes of the facility have not changed; the -interface allows trace functions to be installed per-thread, and the basic -events reported to the trace function are the same as had been reported to the -Python-level trace functions in previous versions. - - -.. c:type:: int (*Py_tracefunc)(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) - - The type of the trace function registered using :c:func:`PyEval_SetProfile` and - :c:func:`PyEval_SetTrace`. The first parameter is the object passed to the - registration function as *obj*, *frame* is the frame object to which the event - pertains, *what* is one of the constants :c:data:`PyTrace_CALL`, - :c:data:`PyTrace_EXCEPTION`, :c:data:`PyTrace_LINE`, :c:data:`PyTrace_RETURN`, - :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION`, :c:data:`PyTrace_C_RETURN`, - or :c:data:`PyTrace_OPCODE`, and *arg* depends on the value of *what*: - - +-------------------------------+----------------------------------------+ - | Value of *what* | Meaning of *arg* | - +===============================+========================================+ - | :c:data:`PyTrace_CALL` | Always :c:data:`Py_None`. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_EXCEPTION` | Exception information as returned by | - | | :func:`sys.exc_info`. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_LINE` | Always :c:data:`Py_None`. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_RETURN` | Value being returned to the caller, | - | | or ``NULL`` if caused by an exception. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_C_CALL` | Function object being called. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_C_EXCEPTION` | Function object being called. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_C_RETURN` | Function object being called. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_OPCODE` | Always :c:data:`Py_None`. | - +-------------------------------+----------------------------------------+ - -.. c:var:: int PyTrace_CALL - - The value of the *what* parameter to a :c:type:`Py_tracefunc` function when a new - call to a function or method is being reported, or a new entry into a generator. - Note that the creation of the iterator for a generator function is not reported - as there is no control transfer to the Python bytecode in the corresponding - frame. - - -.. c:var:: int PyTrace_EXCEPTION - - The value of the *what* parameter to a :c:type:`Py_tracefunc` function when an - exception has been raised. The callback function is called with this value for - *what* when after any bytecode is processed after which the exception becomes - set within the frame being executed. The effect of this is that as exception - propagation causes the Python stack to unwind, the callback is called upon - return to each frame as the exception propagates. Only trace functions receives - these events; they are not needed by the profiler. - - -.. c:var:: int PyTrace_LINE - - The value passed as the *what* parameter to a :c:type:`Py_tracefunc` function - (but not a profiling function) when a line-number event is being reported. - It may be disabled for a frame by setting :attr:`~frame.f_trace_lines` to - *0* on that frame. - - -.. c:var:: int PyTrace_RETURN - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a - call is about to return. - - -.. c:var:: int PyTrace_C_CALL - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function is about to be called. - - -.. c:var:: int PyTrace_C_EXCEPTION - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function has raised an exception. - - -.. c:var:: int PyTrace_C_RETURN - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function has returned. - - -.. c:var:: int PyTrace_OPCODE - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions (but not - profiling functions) when a new opcode is about to be executed. This event is - not emitted by default: it must be explicitly requested by setting - :attr:`~frame.f_trace_opcodes` to *1* on the frame. - - -.. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) - - Set the profiler function to *func*. The *obj* parameter is passed to the - function as its first parameter, and may be any Python object, or ``NULL``. If - the profile function needs to maintain state, using a different value for *obj* - for each thread provides a convenient and thread-safe place to store it. The - profile function is called for all monitored events except :c:data:`PyTrace_LINE` - :c:data:`PyTrace_OPCODE` and :c:data:`PyTrace_EXCEPTION`. - - See also the :func:`sys.setprofile` function. - - The caller must have an :term:`attached thread state`. - -.. c:function:: void PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *obj) - - Like :c:func:`PyEval_SetProfile` but sets the profile function in all running threads - belonging to the current interpreter instead of the setting it only on the current thread. - - The caller must have an :term:`attached thread state`. - - As :c:func:`PyEval_SetProfile`, this function ignores any exceptions raised while - setting the profile functions in all threads. - -.. versionadded:: 3.12 - - -.. c:function:: void PyEval_SetTrace(Py_tracefunc func, PyObject *obj) - - Set the tracing function to *func*. This is similar to - :c:func:`PyEval_SetProfile`, except the tracing function does receive line-number - events and per-opcode events, but does not receive any event related to C function - objects being called. Any trace function registered using :c:func:`PyEval_SetTrace` - will not receive :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION` or - :c:data:`PyTrace_C_RETURN` as a value for the *what* parameter. - - See also the :func:`sys.settrace` function. - - The caller must have an :term:`attached thread state`. - -.. c:function:: void PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *obj) - - Like :c:func:`PyEval_SetTrace` but sets the tracing function in all running threads - belonging to the current interpreter instead of the setting it only on the current thread. - - The caller must have an :term:`attached thread state`. - - As :c:func:`PyEval_SetTrace`, this function ignores any exceptions raised while - setting the trace functions in all threads. - -.. versionadded:: 3.12 - -Reference tracing -================= - -.. versionadded:: 3.13 - -.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) - - The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`. - The first parameter is a Python object that has been just created (when **event** - is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event** - is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer - that was provided when :c:func:`PyRefTracer_SetTracer` was called. - -.. versionadded:: 3.13 - -.. c:var:: int PyRefTracer_CREATE - - The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python - object has been created. - -.. c:var:: int PyRefTracer_DESTROY - - The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python - object has been destroyed. - -.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) - - Register a reference tracer function. The function will be called when a new - Python has been created or when an object is going to be destroyed. If - **data** is provided it must be an opaque pointer that will be provided when - the tracer function is called. Return ``0`` on success. Set an exception and - return ``-1`` on error. - - Not that tracer functions **must not** create Python objects inside or - otherwise the call will be re-entrant. The tracer also **must not** clear - any existing exception or set an exception. A :term:`thread state` will be active - every time the tracer function is called. - - There must be an :term:`attached thread state` when calling this function. - -.. versionadded:: 3.13 - -.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) - - Get the registered reference tracer function and the value of the opaque data - pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called. - If no tracer was registered this function will return NULL and will set the - **data** pointer to NULL. - - There must be an :term:`attached thread state` when calling this function. - -.. versionadded:: 3.13 - -.. _advanced-debugging: - -Advanced Debugger Support -========================= - -.. sectionauthor:: Fred L. Drake, Jr. - - -These functions are only intended to be used by advanced debugging tools. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Head() - - Return the interpreter state object at the head of the list of all such objects. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Main() - - Return the main interpreter state object. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Next(PyInterpreterState *interp) - - Return the next interpreter state object after *interp* from the list of all - such objects. - - -.. c:function:: PyThreadState * PyInterpreterState_ThreadHead(PyInterpreterState *interp) - - Return the pointer to the first :c:type:`PyThreadState` object in the list of - threads associated with the interpreter *interp*. - - -.. c:function:: PyThreadState* PyThreadState_Next(PyThreadState *tstate) - - Return the next thread state object after *tstate* from the list of all such - objects belonging to the same :c:type:`PyInterpreterState` object. - - -.. _thread-local-storage: - -Thread Local Storage Support -============================ - -.. sectionauthor:: Masayuki Yamamoto - -The Python interpreter provides low-level support for thread-local storage -(TLS) which wraps the underlying native TLS implementation to support the -Python-level thread local storage API (:class:`threading.local`). The -CPython C level APIs are similar to those offered by pthreads and Windows: -use a thread key and functions to associate a :c:expr:`void*` value per -thread. - -A :term:`thread state` does *not* need to be :term:`attached ` -when calling these functions; they suppl their own locking. - -Note that :file:`Python.h` does not include the declaration of the TLS APIs, -you need to include :file:`pythread.h` to use thread-local storage. - -.. note:: - None of these API functions handle memory management on behalf of the - :c:expr:`void*` values. You need to allocate and deallocate them yourself. - If the :c:expr:`void*` values happen to be :c:expr:`PyObject*`, these - functions don't do refcount operations on them either. - -.. _thread-specific-storage-api: - -Thread Specific Storage (TSS) API ---------------------------------- - -TSS API is introduced to supersede the use of the existing TLS API within the -CPython interpreter. This API uses a new type :c:type:`Py_tss_t` instead of -:c:expr:`int` to represent thread keys. - -.. versionadded:: 3.7 - -.. seealso:: "A New C-API for Thread-Local Storage in CPython" (:pep:`539`) - - -.. c:type:: Py_tss_t - - This data structure represents the state of a thread key, the definition of - which may depend on the underlying TLS implementation, and it has an - internal field representing the key's initialization state. There are no - public members in this structure. - - When :ref:`Py_LIMITED_API ` is not defined, static allocation of - this type by :c:macro:`Py_tss_NEEDS_INIT` is allowed. - - -.. c:macro:: Py_tss_NEEDS_INIT - - This macro expands to the initializer for :c:type:`Py_tss_t` variables. - Note that this macro won't be defined with :ref:`Py_LIMITED_API `. - - -Dynamic Allocation -~~~~~~~~~~~~~~~~~~ - -Dynamic allocation of the :c:type:`Py_tss_t`, required in extension modules -built with :ref:`Py_LIMITED_API `, where static allocation of this type -is not possible due to its implementation being opaque at build time. - - -.. c:function:: Py_tss_t* PyThread_tss_alloc() - - Return a value which is the same state as a value initialized with - :c:macro:`Py_tss_NEEDS_INIT`, or ``NULL`` in the case of dynamic allocation - failure. - - -.. c:function:: void PyThread_tss_free(Py_tss_t *key) - - Free the given *key* allocated by :c:func:`PyThread_tss_alloc`, after - first calling :c:func:`PyThread_tss_delete` to ensure any associated - thread locals have been unassigned. This is a no-op if the *key* - argument is ``NULL``. - - .. note:: - A freed key becomes a dangling pointer. You should reset the key to - ``NULL``. - - -Methods -~~~~~~~ - -The parameter *key* of these functions must not be ``NULL``. Moreover, the -behaviors of :c:func:`PyThread_tss_set` and :c:func:`PyThread_tss_get` are -undefined if the given :c:type:`Py_tss_t` has not been initialized by -:c:func:`PyThread_tss_create`. - - -.. c:function:: int PyThread_tss_is_created(Py_tss_t *key) - - Return a non-zero value if the given :c:type:`Py_tss_t` has been initialized - by :c:func:`PyThread_tss_create`. - - -.. c:function:: int PyThread_tss_create(Py_tss_t *key) - - Return a zero value on successful initialization of a TSS key. The behavior - is undefined if the value pointed to by the *key* argument is not - initialized by :c:macro:`Py_tss_NEEDS_INIT`. This function can be called - repeatedly on the same key -- calling it on an already initialized key is a - no-op and immediately returns success. - - -.. c:function:: void PyThread_tss_delete(Py_tss_t *key) - - Destroy a TSS key to forget the values associated with the key across all - threads, and change the key's initialization state to uninitialized. A - destroyed key is able to be initialized again by - :c:func:`PyThread_tss_create`. This function can be called repeatedly on - the same key -- calling it on an already destroyed key is a no-op. - - -.. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value) - - Return a zero value to indicate successfully associating a :c:expr:`void*` - value with a TSS key in the current thread. Each thread has a distinct - mapping of the key to a :c:expr:`void*` value. - - -.. c:function:: void* PyThread_tss_get(Py_tss_t *key) - - Return the :c:expr:`void*` value associated with a TSS key in the current - thread. This returns ``NULL`` if no value is associated with the key in the - current thread. - - -.. _thread-local-storage-api: - -Thread Local Storage (TLS) API ------------------------------- - -.. deprecated:: 3.7 - This API is superseded by - :ref:`Thread Specific Storage (TSS) API `. - -.. note:: - This version of the API does not support platforms where the native TLS key - is defined in a way that cannot be safely cast to ``int``. On such platforms, - :c:func:`PyThread_create_key` will return immediately with a failure status, - and the other TLS functions will all be no-ops on such platforms. - -Due to the compatibility problem noted above, this version of the API should not -be used in new code. - -.. c:function:: int PyThread_create_key() -.. c:function:: void PyThread_delete_key(int key) -.. c:function:: int PyThread_set_key_value(int key, void *value) -.. c:function:: void* PyThread_get_key_value(int key) -.. c:function:: void PyThread_delete_key_value(int key) -.. c:function:: void PyThread_ReInitTLS() - -Synchronization Primitives -========================== - -The C-API provides a basic mutual exclusion lock. - -.. c:type:: PyMutex - - A mutual exclusion lock. The :c:type:`!PyMutex` should be initialized to - zero to represent the unlocked state. For example:: - - PyMutex mutex = {0}; - - Instances of :c:type:`!PyMutex` should not be copied or moved. Both the - contents and address of a :c:type:`!PyMutex` are meaningful, and it must - remain at a fixed, writable location in memory. - - .. note:: - - A :c:type:`!PyMutex` currently occupies one byte, but the size should be - considered unstable. The size may change in future Python releases - without a deprecation period. - - .. versionadded:: 3.13 - -.. c:function:: void PyMutex_Lock(PyMutex *m) - - Lock mutex *m*. If another thread has already locked it, the calling - thread will block until the mutex is unlocked. While blocked, the thread - will temporarily detach the :term:`thread state ` if one exists. - - .. versionadded:: 3.13 - -.. c:function:: void PyMutex_Unlock(PyMutex *m) - - Unlock mutex *m*. The mutex must be locked --- otherwise, the function will - issue a fatal error. - - .. versionadded:: 3.13 - -.. c:function:: int PyMutex_IsLocked(PyMutex *m) - - Returns non-zero if the mutex *m* is currently locked, zero otherwise. - - .. note:: - - This function is intended for use in assertions and debugging only and - should not be used to make concurrency control decisions, as the lock - state may change immediately after the check. - - .. versionadded:: 3.14 - -.. _python-critical-section-api: - -Python Critical Section API ---------------------------- - -The critical section API provides a deadlock avoidance layer on top of -per-object locks for :term:`free-threaded ` CPython. They are -intended to replace reliance on the :term:`global interpreter lock`, and are -no-ops in versions of Python with the global interpreter lock. - -Critical sections avoid deadlocks by implicitly suspending active critical -sections and releasing the locks during calls to :c:func:`PyEval_SaveThread`. -When :c:func:`PyEval_RestoreThread` is called, the most recent critical section -is resumed, and its locks reacquired. This means the critical section API -provides weaker guarantees than traditional locks -- they are useful because -their behavior is similar to the :term:`GIL`. - -Variants that accept :c:type:`PyMutex` pointers rather than Python objects are also -available. Use these variants to start a critical section in a situation where -there is no :c:type:`PyObject` -- for example, when working with a C type that -does not extend or wrap :c:type:`PyObject` but still needs to call into the C -API in a manner that might lead to deadlocks. - -The functions and structs used by the macros are exposed for cases -where C macros are not available. They should only be used as in the -given macro expansions. Note that the sizes and contents of the structures may -change in future Python versions. - -.. note:: - - Operations that need to lock two objects at once must use - :c:macro:`Py_BEGIN_CRITICAL_SECTION2`. You *cannot* use nested critical - sections to lock more than one object at once, because the inner critical - section may suspend the outer critical sections. This API does not provide - a way to lock more than two objects at once. - -Example usage:: - - static PyObject * - set_field(MyObject *self, PyObject *value) - { - Py_BEGIN_CRITICAL_SECTION(self); - Py_SETREF(self->field, Py_XNewRef(value)); - Py_END_CRITICAL_SECTION(); - Py_RETURN_NONE; - } - -In the above example, :c:macro:`Py_SETREF` calls :c:macro:`Py_DECREF`, which -can call arbitrary code through an object's deallocation function. The critical -section API avoids potential deadlocks due to reentrancy and lock ordering -by allowing the runtime to temporarily suspend the critical section if the -code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. - -.. c:macro:: Py_BEGIN_CRITICAL_SECTION(op) - - Acquires the per-object lock for the object *op* and begins a - critical section. - - In the free-threaded build, this macro expands to:: - - { - PyCriticalSection _py_cs; - PyCriticalSection_Begin(&_py_cs, (PyObject*)(op)) - - In the default build, this macro expands to ``{``. - - .. versionadded:: 3.13 - -.. c:macro:: Py_BEGIN_CRITICAL_SECTION_MUTEX(m) - - Locks the mutex *m* and begins a critical section. - - In the free-threaded build, this macro expands to:: - - { - PyCriticalSection _py_cs; - PyCriticalSection_BeginMutex(&_py_cs, m) - - Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION`, there is no cast for - the argument of the macro - it must be a :c:type:`PyMutex` pointer. - - On the default build, this macro expands to ``{``. - - .. versionadded:: 3.14 - -.. c:macro:: Py_END_CRITICAL_SECTION() - - Ends the critical section and releases the per-object lock. - - In the free-threaded build, this macro expands to:: - - PyCriticalSection_End(&_py_cs); - } - - In the default build, this macro expands to ``}``. - - .. versionadded:: 3.13 - -.. c:macro:: Py_BEGIN_CRITICAL_SECTION2(a, b) - - Acquires the per-objects locks for the objects *a* and *b* and begins a - critical section. The locks are acquired in a consistent order (lowest - address first) to avoid lock ordering deadlocks. - - In the free-threaded build, this macro expands to:: - - { - PyCriticalSection2 _py_cs2; - PyCriticalSection2_Begin(&_py_cs2, (PyObject*)(a), (PyObject*)(b)) - - In the default build, this macro expands to ``{``. - - .. versionadded:: 3.13 - -.. c:macro:: Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) - - Locks the mutexes *m1* and *m2* and begins a critical section. - - In the free-threaded build, this macro expands to:: - - { - PyCriticalSection2 _py_cs2; - PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) - - Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION2`, there is no cast for - the arguments of the macro - they must be :c:type:`PyMutex` pointers. - - On the default build, this macro expands to ``{``. - - .. versionadded:: 3.14 - -.. c:macro:: Py_END_CRITICAL_SECTION2() - - Ends the critical section and releases the per-object locks. - - In the free-threaded build, this macro expands to:: - - PyCriticalSection2_End(&_py_cs2); - } - - In the default build, this macro expands to ``}``. - - .. versionadded:: 3.13 +- :ref:`initialization` +- :ref:`threads` +- :ref:`synchronization` +- :ref:`thread-local-storage` +- :ref:`sub-interpreter-support` +- :ref:`profiling` diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 24be9ead3874d18..d6b9837987a3999 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -102,7 +102,7 @@ Error Handling * Set *\*err_msg* and return ``1`` if an error is set. * Set *\*err_msg* to ``NULL`` and return ``0`` otherwise. - An error message is an UTF-8 encoded string. + An error message is a UTF-8 encoded string. If *config* has an exit code, format the exit code as an error message. @@ -544,9 +544,9 @@ Configuration Options Visibility: -* Public: Can by get by :c:func:`PyConfig_Get` and set by +* Public: Can be retrieved by :c:func:`PyConfig_Get` and set by :c:func:`PyConfig_Set`. -* Read-only: Can by get by :c:func:`PyConfig_Get`, but cannot be set by +* Read-only: Can be retrieved by :c:func:`PyConfig_Get`, but cannot be set by :c:func:`PyConfig_Set`. @@ -623,6 +623,10 @@ Some options are read from the :mod:`sys` attributes. For example, the option .. versionadded:: 3.14 + .. versionchanged:: next + The function now replaces :data:`sys.flags` (create a new object), + instead of modifying :data:`sys.flags` in-place. + .. _pyconfig_api: @@ -1153,7 +1157,7 @@ PyConfig Most ``PyConfig`` methods :ref:`preinitialize Python ` if needed. In that case, the Python preinitialization configuration - (:c:type:`PyPreConfig`) in based on the :c:type:`PyConfig`. If configuration + (:c:type:`PyPreConfig`) is based on the :c:type:`PyConfig`. If configuration fields which are in common with :c:type:`PyPreConfig` are tuned, they must be set before calling a :c:type:`PyConfig` method: @@ -1278,6 +1282,11 @@ PyConfig Default: ``0``. + .. deprecated-removed:: 3.15 3.17 + + The :option:`-b` and :option:`!-bb` options will become no-op in 3.17. + :c:member:`~PyConfig.bytes_warning` member will be removed in 3.17. + .. c:member:: int warn_default_encoding If non-zero, emit a :exc:`EncodingWarning` warning when :class:`io.TextIOWrapper` @@ -1802,10 +1811,10 @@ PyConfig .. c:member:: wchar_t* run_presite - ``package.module`` path to module that should be imported before - ``site.py`` is run. + ``module`` or ``module:func`` entry point that should be executed before + the :mod:`site` module is imported. - Set by the :option:`-X presite=package.module <-X>` command-line + Set by the :option:`-X presite=module:func <-X>` command-line option and the :envvar:`PYTHON_PRESITE` environment variable. The command-line option takes precedence. @@ -2258,6 +2267,7 @@ If a ``._pth`` file is present: * Set :c:member:`~PyConfig.isolated` to ``1``. * Set :c:member:`~PyConfig.use_environment` to ``0``. * Set :c:member:`~PyConfig.site_import` to ``0``. +* Set :c:member:`~PyConfig.user_site_directory` to ``0`` (since 3.15). * Set :c:member:`~PyConfig.safe_path` to ``1``. If :c:member:`~PyConfig.home` is not set and a ``pyvenv.cfg`` file is present in @@ -2278,6 +2288,12 @@ The ``__PYVENV_LAUNCHER__`` environment variable is used to set therefore affected by :option:`-S`. +.. versionchanged:: 3.15 + + :c:member:`~PyConfig.user_site_directory` is now set to ``0`` when a + ``._pth`` file is present. + + Py_GetArgcArgv() ================ @@ -2287,13 +2303,91 @@ Py_GetArgcArgv() See also :c:member:`PyConfig.orig_argv` member. -Delaying main module execution -============================== -In some embedding use cases, it may be desirable to separate interpreter initialization -from the execution of the main module. +Multi-Phase Initialization Private Provisional API +================================================== + +This section is a private provisional API introducing multi-phase +initialization, the core feature of :pep:`432`: + +* "Core" initialization phase, "bare minimum Python": + + * Builtin types; + * Builtin exceptions; + * Builtin and frozen modules; + * The :mod:`sys` module is only partially initialized + (ex: :data:`sys.path` doesn't exist yet). + +* "Main" initialization phase, Python is fully initialized: + + * Install and configure :mod:`importlib`; + * Apply the :ref:`Path Configuration `; + * Install signal handlers; + * Finish :mod:`sys` module initialization (ex: create :data:`sys.stdout` + and :data:`sys.path`); + * Enable optional features like :mod:`faulthandler` and :mod:`tracemalloc`; + * Import the :mod:`site` module; + * etc. -This separation can be achieved by setting ``PyConfig.run_command`` to the empty -string during initialization (to prevent the interpreter from dropping into the -interactive prompt), and then subsequently executing the desired main module -code using ``__main__.__dict__`` as the global namespace. +Private provisional API: + +.. c:member:: int PyConfig._init_main + + If set to ``0``, :c:func:`Py_InitializeFromConfig` stops at the "Core" + initialization phase. + +.. c:function:: PyStatus _Py_InitializeMain(void) + + Move to the "Main" initialization phase, finish the Python initialization. + +No module is imported during the "Core" phase and the ``importlib`` module is +not configured: the :ref:`Path Configuration ` is only +applied during the "Main" phase. It may allow to customize Python in Python to +override or tune the :ref:`Path Configuration `, maybe +install a custom :data:`sys.meta_path` importer or an import hook, etc. + +It may become possible to calculate the :ref:`Path Configuration +` in Python, after the Core phase and before the Main phase, +which is one of the :pep:`432` motivation. + +The "Core" phase is not properly defined: what should be and what should +not be available at this phase is not specified yet. The API is marked +as private and provisional: the API can be modified or even be removed +anytime until a proper public API is designed. + +Example running Python code between "Core" and "Main" initialization +phases:: + + void init_python(void) + { + PyStatus status; + + PyConfig config; + PyConfig_InitPythonConfig(&config); + config._init_main = 0; + + /* ... customize 'config' configuration ... */ + + status = Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + + /* Use sys.stderr because sys.stdout is only created + by _Py_InitializeMain() */ + int res = PyRun_SimpleString( + "import sys; " + "print('Run Python code before _Py_InitializeMain', " + "file=sys.stderr)"); + if (res < 0) { + exit(1); + } + + /* ... put more configuration code here ... */ + + status = _Py_InitializeMain(); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + } diff --git a/Doc/c-api/interp-lifecycle.rst b/Doc/c-api/interp-lifecycle.rst new file mode 100644 index 000000000000000..46f5b1dd33963c0 --- /dev/null +++ b/Doc/c-api/interp-lifecycle.rst @@ -0,0 +1,974 @@ +.. highlight:: c + +.. _initialization: + +Interpreter initialization and finalization +=========================================== + +See :ref:`Python Initialization Configuration ` for details +on how to configure the interpreter prior to initialization. + +.. _pre-init-safe: + +Before Python initialization +---------------------------- + +In an application embedding Python, the :c:func:`Py_Initialize` function must +be called before using any other Python/C API functions; with the exception of +a few functions and the :ref:`global configuration variables +`. + +The following functions can be safely called before Python is initialized: + +* Functions that initialize the interpreter: + + * :c:func:`Py_Initialize` + * :c:func:`Py_InitializeEx` + * :c:func:`Py_InitializeFromConfig` + * :c:func:`Py_BytesMain` + * :c:func:`Py_Main` + * the runtime pre-initialization functions covered in :ref:`init-config` + +* Configuration functions: + + * :c:func:`PyImport_AppendInittab` + * :c:func:`PyImport_ExtendInittab` + * :c:func:`!PyInitFrozenExtensions` + * :c:func:`PyMem_SetAllocator` + * :c:func:`PyMem_SetupDebugHooks` + * :c:func:`PyObject_SetArenaAllocator` + * :c:func:`Py_SetProgramName` + * :c:func:`Py_SetPythonHome` + * the configuration functions covered in :ref:`init-config` + +* Informative functions: + + * :c:func:`Py_IsInitialized` + * :c:func:`PyMem_GetAllocator` + * :c:func:`PyObject_GetArenaAllocator` + * :c:func:`Py_GetBuildInfo` + * :c:func:`Py_GetCompiler` + * :c:func:`Py_GetCopyright` + * :c:func:`Py_GetPlatform` + * :c:func:`Py_GetVersion` + * :c:func:`Py_IsInitialized` + +* Utilities: + + * :c:func:`Py_DecodeLocale` + * the status reporting and utility functions covered in :ref:`init-config` + +* Memory allocators: + + * :c:func:`PyMem_RawMalloc` + * :c:func:`PyMem_RawRealloc` + * :c:func:`PyMem_RawCalloc` + * :c:func:`PyMem_RawFree` + +* Synchronization: + + * :c:func:`PyMutex_Lock` + * :c:func:`PyMutex_Unlock` + +.. note:: + + Despite their apparent similarity to some of the functions listed above, + the following functions **should not be called** before the interpreter has + been initialized: :c:func:`Py_EncodeLocale`, :c:func:`PyEval_InitThreads`, and + :c:func:`Py_RunMain`. + + +.. _global-conf-vars: + +Global configuration variables +------------------------------ + +Python has variables for the global configuration to control different features +and options. By default, these flags are controlled by :ref:`command line +options `. + +When a flag is set by an option, the value of the flag is the number of times +that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` +to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. + + +.. c:var:: int Py_BytesWarningFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.bytes_warning` should be used instead, see :ref:`Python + Initialization Configuration `. + + Issue a warning when comparing :class:`bytes` or :class:`bytearray` with + :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater + or equal to ``2``. + + Set by the :option:`-b` option. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_DebugFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.parser_debug` should be used instead, see :ref:`Python + Initialization Configuration `. + + Turn on parser debugging output (for expert only, depending on compilation + options). + + Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment + variable. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_DontWriteBytecodeFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.write_bytecode` should be used instead, see :ref:`Python + Initialization Configuration `. + + If set to non-zero, Python won't try to write ``.pyc`` files on the + import of source modules. + + Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` + environment variable. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_FrozenFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.pathconfig_warnings` should be used instead, see + :ref:`Python Initialization Configuration `. + + Private flag used by ``_freeze_module`` and ``frozenmain`` programs. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_HashRandomizationFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.hash_seed` and :c:member:`PyConfig.use_hash_seed` should + be used instead, see :ref:`Python Initialization Configuration + `. + + Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to + a non-empty string. + + If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment + variable to initialize the secret hash seed. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_IgnoreEnvironmentFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.use_environment` should be used instead, see + :ref:`Python Initialization Configuration `. + + Ignore all :envvar:`!PYTHON*` environment variables, e.g. + :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. + + Set by the :option:`-E` and :option:`-I` options. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_InspectFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.inspect` should be used instead, see + :ref:`Python Initialization Configuration `. + + When a script is passed as first argument or the :option:`-c` option is used, + enter interactive mode after executing the script or the command, even when + :data:`sys.stdin` does not appear to be a terminal. + + Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment + variable. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_InteractiveFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.interactive` should be used instead, see + :ref:`Python Initialization Configuration `. + + Set by the :option:`-i` option. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_IsolatedFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.isolated` should be used instead, see + :ref:`Python Initialization Configuration `. + + Run Python in isolated mode. In isolated mode :data:`sys.path` contains + neither the script's directory nor the user's site-packages directory. + + Set by the :option:`-I` option. + + .. versionadded:: 3.4 + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_LegacyWindowsFSEncodingFlag + + This API is kept for backward compatibility: setting + :c:member:`PyPreConfig.legacy_windows_fs_encoding` should be used instead, see + :ref:`Python Initialization Configuration `. + + If the flag is non-zero, use the ``mbcs`` encoding with ``replace`` error + handler, instead of the UTF-8 encoding with ``surrogatepass`` error handler, + for the :term:`filesystem encoding and error handler`. + + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment + variable is set to a non-empty string. + + See :pep:`529` for more details. + + .. availability:: Windows. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_LegacyWindowsStdioFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.legacy_windows_stdio` should be used instead, see + :ref:`Python Initialization Configuration `. + + If the flag is non-zero, use :class:`io.FileIO` instead of + :class:`!io._WindowsConsoleIO` for :mod:`sys` standard streams. + + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment + variable is set to a non-empty string. + + See :pep:`528` for more details. + + .. availability:: Windows. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_NoSiteFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.site_import` should be used instead, see + :ref:`Python Initialization Configuration `. + + Disable the import of the module :mod:`site` and the site-dependent + manipulations of :data:`sys.path` that it entails. Also disable these + manipulations if :mod:`site` is explicitly imported later (call + :func:`site.main` if you want them to be triggered). + + Set by the :option:`-S` option. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_NoUserSiteDirectory + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.user_site_directory` should be used instead, see + :ref:`Python Initialization Configuration `. + + Don't add the :data:`user site-packages directory ` to + :data:`sys.path`. + + Set by the :option:`-s` and :option:`-I` options, and the + :envvar:`PYTHONNOUSERSITE` environment variable. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_OptimizeFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.optimization_level` should be used instead, see + :ref:`Python Initialization Configuration `. + + Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment + variable. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_QuietFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.quiet` should be used instead, see :ref:`Python + Initialization Configuration `. + + Don't display the copyright and version messages even in interactive mode. + + Set by the :option:`-q` option. + + .. versionadded:: 3.2 + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_UnbufferedStdioFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.buffered_stdio` should be used instead, see :ref:`Python + Initialization Configuration `. + + Force the stdout and stderr streams to be unbuffered. + + Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` + environment variable. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_VerboseFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.verbose` should be used instead, see :ref:`Python + Initialization Configuration `. + + Print a message each time a module is initialized, showing the place + (filename or built-in module) from which it is loaded. If greater or equal + to ``2``, print a message for each file that is checked for when + searching for a module. Also provides information on module cleanup at exit. + + Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment + variable. + + .. deprecated-removed:: 3.12 3.16 + + +Initializing and finalizing the interpreter +------------------------------------------- + +.. c:function:: void Py_Initialize() + + .. index:: + single: PyEval_InitThreads() + single: modules (in module sys) + single: path (in module sys) + pair: module; builtins + pair: module; __main__ + pair: module; sys + triple: module; search; path + single: Py_FinalizeEx (C function) + + Initialize the Python interpreter. In an application embedding Python, + this should be called before using any other Python/C API functions; see + :ref:`Before Python Initialization ` for the few exceptions. + + This initializes the table of loaded modules (``sys.modules``), and creates + the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. + It also initializes the module search path (``sys.path``). It does not set + ``sys.argv``; use the :ref:`Python Initialization Configuration ` + API for that. This is a no-op when called for a second time (without calling + :c:func:`Py_FinalizeEx` first). There is no return value; it is a fatal + error if the initialization fails. + + Use :c:func:`Py_InitializeFromConfig` to customize the + :ref:`Python Initialization Configuration `. + + .. note:: + On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, + which will also affect non-Python uses of the console using the C Runtime. + + +.. c:function:: void Py_InitializeEx(int initsigs) + + This function works like :c:func:`Py_Initialize` if *initsigs* is ``1``. If + *initsigs* is ``0``, it skips initialization registration of signal handlers, + which may be useful when CPython is embedded as part of a larger application. + + Use :c:func:`Py_InitializeFromConfig` to customize the + :ref:`Python Initialization Configuration `. + + +.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) + + Initialize Python from *config* configuration, as described in + :ref:`init-from-config`. + + See the :ref:`init-config` section for details on pre-initializing the + interpreter, populating the runtime configuration structure, and querying + the returned status structure. + + +.. c:function:: int Py_IsInitialized() + + Return true (nonzero) when the Python interpreter has been initialized, false + (zero) if not. After :c:func:`Py_FinalizeEx` is called, this returns false until + :c:func:`Py_Initialize` is called again. + + .. versionchanged:: 3.15 + This function no longer returns true until initialization has fully + completed, including import of the :mod:`site` module. Previously it + could return true while :c:func:`Py_Initialize` was still running. + + +.. c:function:: int Py_IsFinalizing() + + Return true (non-zero) if the main Python interpreter is + :term:`shutting down `. Return false (zero) otherwise. + + .. versionadded:: 3.13 + + +.. c:function:: int Py_FinalizeEx() + + Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of + Python/C API functions, and destroy all sub-interpreters (see + :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since + the last call to :c:func:`Py_Initialize`. This is a no-op when called for a second + time (without calling :c:func:`Py_Initialize` again first). + + Since this is the reverse of :c:func:`Py_Initialize`, it should be called + in the same thread with the same interpreter active. That means + the main thread and the main interpreter. + This should never be called while :c:func:`Py_RunMain` is running. + + Normally the return value is ``0``. + If there were errors during finalization (flushing buffered data), + ``-1`` is returned. + + Note that Python will do a best effort at freeing all memory allocated by the Python + interpreter. Therefore, any C-Extension should make sure to correctly clean up all + of the previously allocated PyObjects before using them in subsequent calls to + :c:func:`Py_Initialize`. Otherwise it could introduce vulnerabilities and incorrect + behavior. + + This function is provided for a number of reasons. An embedding application + might want to restart Python without having to restart the application itself. + An application that has loaded the Python interpreter from a dynamically + loadable library (or DLL) might want to free all memory allocated by Python + before unloading the DLL. During a hunt for memory leaks in an application a + developer might want to free all memory allocated by Python before exiting from + the application. + + **Bugs and caveats:** The destruction of modules and objects in modules is done + in random order; this may cause destructors (:meth:`~object.__del__` methods) to fail + when they depend on other objects (even functions) or modules. Dynamically + loaded extension modules loaded by Python are not unloaded. Small amounts of + memory allocated by the Python interpreter may not be freed (if you find a leak, + please report it). Memory tied up in circular references between objects is not + freed. Interned strings will all be deallocated regardless of their reference count. + Some memory allocated by extension modules may not be freed. Some extensions may not + work properly if their initialization routine is called more than once; this can + happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` + more than once. :c:func:`Py_FinalizeEx` must not be called recursively from + within itself. Therefore, it must not be called by any code that may be run + as part of the interpreter shutdown process, such as :py:mod:`atexit` + handlers, object finalizers, or any code that may be run while flushing the + stdout and stderr files. + + .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx + + .. versionadded:: 3.6 + + +.. c:function:: void Py_Finalize() + + This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that + disregards the return value. + + +.. c:function:: int Py_BytesMain(int argc, char **argv) + + Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings, + allowing the calling application to delegate the text decoding step to + the CPython runtime. + + .. versionadded:: 3.8 + + +.. c:function:: int Py_Main(int argc, wchar_t **argv) + + The main program for the standard interpreter, encapsulating a full + initialization/finalization cycle, as well as additional + behaviour to implement reading configurations settings from the environment + and command line, and then executing ``__main__`` in accordance with + :ref:`using-on-cmdline`. + + This is made available for programs which wish to support the full CPython + command line interface, rather than just embedding a Python runtime in a + larger application. + + The *argc* and *argv* parameters are similar to those which are passed to a + C program's :c:func:`main` function, except that the *argv* entries are first + converted to ``wchar_t`` using :c:func:`Py_DecodeLocale`. It is also + important to note that the argument list entries may be modified to point to + strings other than those passed in (however, the contents of the strings + pointed to by the argument list are not modified). + + The return value is ``2`` if the argument list does not represent a valid + Python command line, and otherwise the same as :c:func:`Py_RunMain`. + + In terms of the CPython runtime configuration APIs documented in the + :ref:`runtime configuration ` section (and without accounting + for error handling), ``Py_Main`` is approximately equivalent to:: + + PyConfig config; + PyConfig_InitPythonConfig(&config); + PyConfig_SetArgv(&config, argc, argv); + Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + + Py_RunMain(); + + In normal usage, an embedding application will call this function + *instead* of calling :c:func:`Py_Initialize`, :c:func:`Py_InitializeEx` or + :c:func:`Py_InitializeFromConfig` directly, and all settings will be applied + as described elsewhere in this documentation. If this function is instead + called *after* a preceding runtime initialization API call, then exactly + which environmental and command line configuration settings will be updated + is version dependent (as it depends on which settings correctly support + being modified after they have already been set once when the runtime was + first initialized). + + +.. c:function:: int Py_RunMain(void) + + Executes the main module in a fully configured CPython runtime. + + Executes the command (:c:member:`PyConfig.run_command`), the script + (:c:member:`PyConfig.run_filename`) or the module + (:c:member:`PyConfig.run_module`) specified on the command line or in the + configuration. If none of these values are set, runs the interactive Python + prompt (REPL) using the ``__main__`` module's global namespace. + + If :c:member:`PyConfig.inspect` is not set (the default), the return value + will be ``0`` if the interpreter exits normally (that is, without raising + an exception), the exit status of an unhandled :exc:`SystemExit`, or ``1`` + for any other unhandled exception. + + If :c:member:`PyConfig.inspect` is set (such as when the :option:`-i` option + is used), rather than returning when the interpreter exits, execution will + instead resume in an interactive Python prompt (REPL) using the ``__main__`` + module's global namespace. If the interpreter exited with an exception, it + is immediately raised in the REPL session. The function return value is + then determined by the way the *REPL session* terminates: ``0``, ``1``, or + the status of a :exc:`SystemExit`, as specified above. + + This function always finalizes the Python interpreter before it returns. + + See :ref:`Python Configuration ` for an example of a + customized Python that always runs in isolated mode using + :c:func:`Py_RunMain`. + +.. c:function:: int PyUnstable_AtExit(PyInterpreterState *interp, void (*func)(void *), void *data) + + Register an :mod:`atexit` callback for the target interpreter *interp*. + This is similar to :c:func:`Py_AtExit`, but takes an explicit interpreter and + data pointer for the callback. + + There must be an :term:`attached thread state` for *interp*. + + .. versionadded:: 3.13 + + +.. _cautions-regarding-runtime-finalization: + +Cautions regarding interpreter finalization +------------------------------------------- + +In the late stage of :term:`interpreter shutdown`, after attempting to wait for +non-daemon threads to exit (though this can be interrupted by +:class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime +is marked as finalizing, meaning that :c:func:`Py_IsFinalizing` and +:func:`sys.is_finalizing` return true. At this point, only the finalization +thread (the thread that initiated finalization; this is typically the main thread) +is allowed to :term:`attach ` a thread state. + +Other threads that attempt to attach during finalization, either explicitly +(such as via :c:func:`PyThreadState_Ensure` or :c:macro:`Py_END_ALLOW_THREADS`) +or implicitly (such as in-between bytecode instructions), will enter a +**permanently blocked state**. Generally, this is harmless, but this can +result in deadlocks. For example, a thread may be permanently blocked while +holding a lock, meaning that the finalization thread can never acquire that +lock. + +Prior to CPython 3.13, the thread would exit instead of hanging, +which led to other issues (see the warning note at +:c:func:`PyThread_exit_thread`). + +Gross? Yes. Starting in Python 3.15, there are a number of C APIs that make +it possible to avoid these issues by temporarily preventing finalization: + +.. _interpreter-guards: + +.. seealso:: + + :pep:`788` explains the design, motivation and rationale + for these APIs. + +.. c:type:: PyInterpreterGuard + + An opaque interpreter guard structure. + + By holding an interpreter guard, the caller can ensure that the interpreter + will not finalize until the guard is closed (through + :c:func:`PyInterpreterGuard_Close`). + + When a guard is held, a thread attempting to finalize the interpreter will + block until the guard is closed before starting finalization. + After finalization has started, threads are forever unable to acquire + guards for that interpreter. This means that if you forget to close an + interpreter guard, the process will **permanently hang** during + finalization! + + Holding a guard for an interpreter is similar to holding a + :term:`strong reference` to a Python object, except finalization does not happen + automatically after all guards are released: it requires an explicit + :c:func:`Py_EndInterpreter` call. + + .. versionadded:: 3.15 + + +.. c:function:: PyInterpreterGuard *PyInterpreterGuard_FromCurrent(void) + + Create a finalization guard for the current interpreter. This will prevent + finalization until the guard is closed. + + For example: + + .. code-block:: c + + // Temporarily prevent finalization. + PyInterpreterGuard *guard = PyInterpreterGuard_FromCurrent(); + if (guard == NULL) { + // Finalization has already started or we're out of memory. + return NULL; + } + + Py_BEGIN_ALLOW_THREADS; + // Do some critical processing here. For example, we can safely acquire + // locks that might be acquired by the finalization thread. + Py_END_ALLOW_THREADS; + + // Now that we're done with our critical processing, the interpreter is + // allowed to finalize again. + PyInterpreterGuard_Close(guard); + + On success, this function returns a guard for the current interpreter; + on failure, it returns ``NULL`` with an exception set. + + This function will fail only if the current interpreter has already started + finalizing, or if the process is out of memory. + + The guard pointer returned by this function must be eventually closed + with :c:func:`PyInterpreterGuard_Close`; failing to do so will result in + the Python process infinitely hanging. + + The caller must hold an :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. c:function:: PyInterpreterGuard *PyInterpreterGuard_FromView(PyInterpreterView *view) + + Create a finalization guard for an interpreter through a view. + + On success, this function returns a guard to the interpreter + represented by *view*. The view is still valid after calling this + function. The guard must eventually be closed with + :c:func:`PyInterpreterGuard_Close`. + + If the interpreter no longer exists, is already finalizing, or out of memory, + then this function returns ``NULL`` without setting an exception. + + The caller does not need to hold an :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. c:function:: void PyInterpreterGuard_Close(PyInterpreterGuard *guard) + + Close an interpreter guard, allowing the interpreter to start + finalization if no other guards remain. If an interpreter guard + is never closed, the interpreter will infinitely wait when trying + to enter finalization! + + After an interpreter guard is closed, it may not be used in + :c:func:`PyThreadState_Ensure`. Doing so will result in undefined + behavior. + + This function cannot fail, and the caller doesn't need to hold an + :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. _interpreter-views: + +Interpreter views +----------------- + +In some cases, it may be necessary to access an interpreter that may have been +deleted. This can be done using interpreter views. + +.. c:type:: PyInterpreterView + + An opaque view of an interpreter. + + This is a thread-safe way to access an interpreter that may have be + finalizing or already destroyed. + + .. versionadded:: 3.15 + + +.. c:function:: PyInterpreterView *PyInterpreterView_FromCurrent(void) + + Create a view to the current interpreter. + + This function is generally meant to be used alongside + :c:func:`PyInterpreterGuard_FromView` or :c:func:`PyThreadState_EnsureFromView`. + + On success, this function returns a view to the current interpreter; on + failure, it returns ``NULL`` with an exception set. + + The caller must hold an :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. c:function:: void PyInterpreterView_Close(PyInterpreterView *view) + + Close an interpreter view. + + If an interpreter view is never closed, the view's memory will never be + freed, but there are no other consequences. (In contrast, forgetting to + close a guard will infinitely hang the main thread during finalization.) + + This function cannot fail, and the caller doesn't need to hold an + :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. c:function:: PyInterpreterView *PyInterpreterView_FromMain(void) + + Create a view for the main interpreter (the first and default + interpreter in a Python process; see + :c:func:`PyInterpreterState_Main`). + + On success, this function returns a view to the main + interpreter; on failure, it returns ``NULL`` without an exception set. + Failure indicates that the process is out of memory. + + Use this function when an interpreter pointer or view cannot be supplied + by the caller, such as when a native threading library does not provide a + ``void *arg`` parameter that could carry a :c:type:`PyInterpreterGuard` or + :c:type:`PyInterpreterView`. In code that supports subinterpreters, prefer + :c:func:`PyInterpreterView_FromCurrent` so the guard tracks the calling + interpreter rather than the main one. + + The caller does not need to hold an :term:`attached thread state`. + + .. versionadded:: 3.15 + + +Process-wide parameters +----------------------- + +.. c:function:: void Py_SetProgramName(const wchar_t *name) + + .. index:: + single: Py_Initialize() + single: main() + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python + Initialization Configuration `. + + This function should be called before :c:func:`Py_Initialize` is called for + the first time, if it is called at all. It tells the interpreter the value + of the ``argv[0]`` argument to the :c:func:`main` function of the program + (converted to wide characters). + This is used by some other functions below to find + the Python run-time libraries relative to the interpreter executable. The + default value is ``'python'``. The argument should point to a + zero-terminated wide character string in static storage whose contents will not + change for the duration of the program's execution. No code in the Python + interpreter will change the contents of this storage. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + .. deprecated-removed:: 3.11 3.16 + + +.. c:function:: const char* Py_GetVersion() + + Return the version of this Python interpreter. This is a string that looks + something like :: + + "3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]" + + .. index:: single: version (in module sys) + + The first word (up to the first space character) is the current Python version; + the first characters are the major and minor version separated by a + period. The returned string points into static storage; the caller should not + modify its value. The value is available to Python code as :data:`sys.version`. + + See also the :c:var:`Py_Version` constant. + + +.. c:function:: const char* Py_GetPlatform() + + .. index:: single: platform (in module sys) + + Return the platform identifier for the current platform. On Unix, this is + formed from the "official" name of the operating system, converted to lower + case, followed by the major revision number; e.g., for Solaris 2.x, which is + also known as SunOS 5.x, the value is ``'sunos5'``. On macOS, it is + ``'darwin'``. On Windows, it is ``'win'``. The returned string points into + static storage; the caller should not modify its value. The value is available + to Python code as ``sys.platform``. + + +.. c:function:: const char* Py_GetCopyright() + + Return the official copyright string for the current Python version, for example + + ``'Copyright 1991-1995 Stichting Mathematisch Centrum, Amsterdam'`` + + .. index:: single: copyright (in module sys) + + The returned string points into static storage; the caller should not modify its + value. The value is available to Python code as ``sys.copyright``. + + +.. c:function:: const char* Py_GetCompiler() + + Return an indication of the compiler used to build the current Python version, + in square brackets, for example:: + + "[GCC 2.7.2.2]" + + .. index:: single: version (in module sys) + + The returned string points into static storage; the caller should not modify its + value. The value is available to Python code as part of the variable + ``sys.version``. + + +.. c:function:: const char* Py_GetBuildInfo() + + Return information about the sequence number and build date and time of the + current Python interpreter instance, for example :: + + "#67, Aug 1 1997, 22:34:28" + + .. index:: single: version (in module sys) + + The returned string points into static storage; the caller should not modify its + value. The value is available to Python code as part of the variable + ``sys.version``. + + +.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) + + .. index:: + single: main() + single: Py_FatalError() + single: argv (in module sys) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.argv`, :c:member:`PyConfig.parse_argv` and + :c:member:`PyConfig.safe_path` should be used instead, see :ref:`Python + Initialization Configuration `. + + Set :data:`sys.argv` based on *argc* and *argv*. These parameters are + similar to those passed to the program's :c:func:`main` function with the + difference that the first entry should refer to the script file to be + executed rather than the executable hosting the Python interpreter. If there + isn't a script that will be run, the first entry in *argv* can be an empty + string. If this function fails to initialize :data:`sys.argv`, a fatal + condition is signalled using :c:func:`Py_FatalError`. + + If *updatepath* is zero, this is all the function does. If *updatepath* + is non-zero, the function also modifies :data:`sys.path` according to the + following algorithm: + + - If the name of an existing script is passed in ``argv[0]``, the absolute + path of the directory where the script is located is prepended to + :data:`sys.path`. + - Otherwise (that is, if *argc* is ``0`` or ``argv[0]`` doesn't point + to an existing file name), an empty string is prepended to + :data:`sys.path`, which is the same as prepending the current working + directory (``"."``). + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` + members of the :ref:`Python Initialization Configuration `. + + .. note:: + It is recommended that applications embedding the Python interpreter + for purposes other than executing a single script pass ``0`` as *updatepath*, + and update :data:`sys.path` themselves if desired. + See :cve:`2008-5983`. + + On versions before 3.1.3, you can achieve the same effect by manually + popping the first :data:`sys.path` element after having called + :c:func:`PySys_SetArgv`, for example using:: + + PyRun_SimpleString("import sys; sys.path.pop(0)\n"); + + .. versionadded:: 3.1.3 + + .. deprecated-removed:: 3.11 3.16 + + +.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.argv` and :c:member:`PyConfig.parse_argv` should be used + instead, see :ref:`Python Initialization Configuration `. + + This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set + to ``1`` unless the :program:`python` interpreter was started with the + :option:`-I`. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` + members of the :ref:`Python Initialization Configuration `. + + .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. + + .. deprecated-removed:: 3.11 3.16 + + +.. c:function:: void Py_SetPythonHome(const wchar_t *home) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.home` should be used instead, see :ref:`Python + Initialization Configuration `. + + Set the default "home" directory, that is, the location of the standard + Python libraries. See :envvar:`PYTHONHOME` for the meaning of the + argument string. + + The argument should point to a zero-terminated character string in static + storage whose contents will not change for the duration of the program's + execution. No code in the Python interpreter will change the contents of + this storage. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + .. deprecated-removed:: 3.11 3.16 diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index acce3dc215d1573..500f2818e2e40a6 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -107,6 +107,46 @@ header files properly declare the entry points to be ``extern "C"``. As a result there is no need to do anything special to use the API from C++. +.. _capi-system-includes: + +System includes +--------------- + + :file:`Python.h` includes several standard header files. + C extensions should include the standard headers that they use, + and should not rely on these implicit includes. + The implicit includes are: + + * ```` + * ```` (on Windows) + * ```` + * ```` + * ```` + * ```` + * ```` + * ```` + * ```` (if present) + + The following are included for backwards compatibility, unless using + :ref:`Limited API ` 3.13 or newer: + + * ```` + * ```` (on POSIX) + + The following are included for backwards compatibility, unless using + :ref:`Limited API ` 3.11 or newer: + + * ```` + * ```` + * ```` + +.. note:: + + Since Python may define some pre-processor definitions which affect the standard + headers on some systems, you *must* include :file:`Python.h` before any standard + headers are included. + + Useful macros ============= @@ -116,18 +156,279 @@ defined closer to where they are useful (for example, :c:macro:`Py_RETURN_NONE`, Others of a more general utility are defined here. This is not necessarily a complete listing. +.. c:macro:: Py_CAN_START_THREADS + + If this macro is defined, then the current system is able to start threads. + + Currently, all systems supported by CPython (per :pep:`11`), with the + exception of some WebAssembly platforms, support starting threads. + + .. versionadded:: 3.13 + +.. c:macro:: Py_GETENV(s) + + Like :samp:`getenv({s})`, but returns ``NULL`` if :option:`-E` was passed + on the command line (see :c:member:`PyConfig.use_environment`). + + +Docstring macros +---------------- + +.. c:macro:: PyDoc_STRVAR(name, str) + + Creates a variable with name *name* that can be used in docstrings. + If Python is built without docstrings (:option:`--without-doc-strings`), + the value will be an empty string. + + Example:: + + PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element."); + + static PyMethodDef deque_methods[] = { + // ... + {"pop", (PyCFunction)deque_pop, METH_NOARGS, pop_doc}, + // ... + } + + Expands to :samp:`PyDoc_VAR({name}) = PyDoc_STR({str})`. + +.. c:macro:: PyDoc_STR(str) + + Expands to the given input string, or an empty string + if docstrings are disabled (:option:`--without-doc-strings`). + + Example:: + + static PyMethodDef pysqlite_row_methods[] = { + {"keys", (PyCFunction)pysqlite_row_keys, METH_NOARGS, + PyDoc_STR("Returns the keys of the row.")}, + {NULL, NULL} + }; + +.. c:macro:: PyDoc_VAR(name) + + Declares a static character array variable with the given *name*. + Expands to :samp:`static const char {name}[]` + + For example:: + + PyDoc_VAR(python_doc) = PyDoc_STR( + "A genus of constricting snakes in the Pythonidae family native " + "to the tropics and subtropics of the Eastern Hemisphere."); + + +General utility macros +---------------------- + +The following macros are for common tasks not specific to Python. + +.. c:macro:: Py_UNUSED(arg) + + Use this for unused arguments in a function definition to silence compiler + warnings. Example: ``int func(int a, int Py_UNUSED(b)) { return a; }``. + + .. versionadded:: 3.4 + +.. c:macro:: Py_GCC_ATTRIBUTE(name) + + Use a GCC attribute *name*, hiding it from compilers that don't support GCC + attributes (such as MSVC). + + This expands to :samp:`__attribute__(({name)})` on a GCC compiler, + and expands to nothing on compilers that don't support GCC attributes. + + +Numeric utilities +^^^^^^^^^^^^^^^^^ .. c:macro:: Py_ABS(x) Return the absolute value of ``x``. + The argument may be evaluated more than once. + Consequently, do not pass an expression with side-effects directly + to this macro. + + If the result cannot be represented (for example, if ``x`` has + :c:macro:`!INT_MIN` value for :c:expr:`int` type), the behavior is + undefined. + + Corresponds roughly to :samp:`(({x}) < 0 ? -({x}) : ({x}))` + + .. versionadded:: 3.3 + +.. c:macro:: Py_MAX(x, y) + Py_MIN(x, y) + + Return the larger or smaller of the arguments, respectively. + + Any arguments may be evaluated more than once. + Consequently, do not pass an expression with side-effects directly + to this macro. + + :c:macro:`!Py_MAX` corresponds roughly to + :samp:`((({x}) > ({y})) ? ({x}) : ({y}))`. + .. versionadded:: 3.3 +.. c:macro:: Py_ARITHMETIC_RIGHT_SHIFT(type, integer, positions) + + Similar to :samp:`{integer} >> {positions}`, but forces sign extension, + as the C standard does not define whether a right-shift of a signed + integer will perform sign extension or a zero-fill. + + *integer* should be any signed integer type. + *positions* is the number of positions to shift to the right. + + Both *integer* and *positions* can be evaluated more than once; + consequently, avoid directly passing a function call or some other + operation with side-effects to this macro. Instead, store the result as a + variable and then pass it. + + *type* is unused and only kept for backwards compatibility. Historically, + *type* was used to cast *integer*. + + .. versionchanged:: 3.1 + + This macro is now valid for all signed integer types, not just those for + which ``unsigned type`` is legal. As a result, *type* is no longer + used. + +.. c:macro:: Py_CHARMASK(c) + + Argument must be a character or an integer in the range [-128, 127] or [0, + 255]. This macro returns ``c`` cast to an ``unsigned char``. + + +Assertion utilities +^^^^^^^^^^^^^^^^^^^ + +.. c:macro:: Py_UNREACHABLE() + + Use this when you have a code path that cannot be reached by design. + For example, in the ``default:`` clause in a ``switch`` statement for which + all possible values are covered in ``case`` statements. Use this in places + where you might be tempted to put an ``assert(0)`` or ``abort()`` call. + + In release mode, the macro helps the compiler to optimize the code, and + avoids a warning about unreachable code. For example, the macro is + implemented with ``__builtin_unreachable()`` on GCC in release mode. + + In debug mode, and on unsupported compilers, the macro expands to a call to + :c:func:`Py_FatalError`. + + A use for ``Py_UNREACHABLE()`` is following a call to a function that + never returns but that is not declared ``_Noreturn``. + + If a code path is very unlikely code but can be reached under exceptional + case, this macro must not be used. For example, under low memory condition + or if a system call returns a value out of the expected range. In this + case, it's better to report the error to the caller. If the error cannot + be reported to caller, :c:func:`Py_FatalError` can be used. + + .. versionadded:: 3.7 + +.. c:macro:: Py_SAFE_DOWNCAST(value, larger, smaller) + + Cast *value* to type *smaller* from type *larger*, validating that no + information was lost. + + On release builds of Python, this is roughly equivalent to + :samp:`(({smaller}) {value})` + (in C++, :samp:`static_cast<{smaller}>({value})` will be used instead). + + On debug builds (implying that :c:macro:`Py_DEBUG` is defined), this asserts + that no information was lost with the cast from *larger* to *smaller*. + + *value*, *larger*, and *smaller* may all be evaluated more than once in the + expression; consequently, do not pass an expression with side-effects + directly to this macro. + +.. c:macro:: Py_BUILD_ASSERT(cond) + + Asserts a compile-time condition *cond*, as a statement. + The build will fail if the condition is false or cannot be evaluated at compile time. + + Corresponds roughly to :samp:`static_assert({cond})` on C23 and above. + + For example:: + + Py_BUILD_ASSERT(sizeof(PyTime_t) == sizeof(int64_t)); + + .. versionadded:: 3.3 + +.. c:macro:: Py_BUILD_ASSERT_EXPR(cond) + + Asserts a compile-time condition *cond*, as an expression that evaluates to ``0``. + The build will fail if the condition is false or cannot be evaluated at compile time. + + For example:: + + #define foo_to_char(foo) \ + ((char *)(foo) + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) + + .. versionadded:: 3.3 + + +Type size utilities +^^^^^^^^^^^^^^^^^^^ + +.. c:macro:: Py_ARRAY_LENGTH(array) + + Compute the length of a statically allocated C array at compile time. + + The *array* argument must be a C array with a size known at compile time. + Passing an array with an unknown size, such as a heap-allocated array, + will result in a compilation error on some compilers, or otherwise produce + incorrect results. + + This is roughly equivalent to:: + + sizeof(array) / sizeof((array)[0]) + +.. c:macro:: Py_MEMBER_SIZE(type, member) + + Return the size of a structure (*type*) *member* in bytes. + + Corresponds roughly to :samp:`sizeof((({type} *)NULL)->{member})`. + + .. versionadded:: 3.6 + + +Macro definition utilities +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:macro:: Py_FORCE_EXPANSION(X) + + This is equivalent to :samp:`{X}`, which is useful for token-pasting in + macros, as macro expansions in *X* are forcefully evaluated by the + preprocessor. + +.. c:macro:: Py_STRINGIFY(x) + + Convert ``x`` to a C string. For example, ``Py_STRINGIFY(123)`` returns + ``"123"``. + + .. versionadded:: 3.4 + + +Declaration utilities +--------------------- + +The following macros can be used in declarations. +They are most useful for defining the C API itself, and have limited use +for extension authors. +Most of them expand to compiler-specific spellings of common extensions +to the C language. + .. c:macro:: Py_ALWAYS_INLINE Ask the compiler to always inline a static inline function. The compiler can ignore it and decide to not inline the function. + Corresponds to ``always_inline`` attribute in GCC and ``__forceinline`` + in MSVC. + It can be used to inline performance critical static inline functions when building Python in debug mode with function inlining disabled. For example, MSC disables function inlining when building in debug mode. @@ -145,15 +446,24 @@ complete listing. .. versionadded:: 3.11 -.. c:macro:: Py_CHARMASK(c) +.. c:macro:: Py_NO_INLINE - Argument must be a character or an integer in the range [-128, 127] or [0, - 255]. This macro returns ``c`` cast to an ``unsigned char``. + Disable inlining on a function. For example, it reduces the C stack + consumption: useful on LTO+PGO builds which heavily inline code (see + :issue:`33720`). + + Corresponds to the ``noinline`` attribute/specification on GCC and MSVC. + + Usage:: + + Py_NO_INLINE static int random(void) { return 4; } + + .. versionadded:: 3.11 .. c:macro:: Py_DEPRECATED(version) - Use this for deprecated declarations. The macro must be placed before the - symbol name. + Use this to declare APIs that were deprecated in a specific CPython version. + The macro must be placed before the symbol name. Example:: @@ -162,110 +472,153 @@ complete listing. .. versionchanged:: 3.8 MSVC support was added. -.. c:macro:: Py_GETENV(s) +.. c:macro:: Py_LOCAL(type) - Like ``getenv(s)``, but returns ``NULL`` if :option:`-E` was passed on the - command line (see :c:member:`PyConfig.use_environment`). + Declare a function returning the specified *type* using a fast-calling + qualifier for functions that are local to the current file. + Semantically, this is equivalent to :samp:`static {type}`. -.. c:macro:: Py_MAX(x, y) +.. c:macro:: Py_LOCAL_INLINE(type) - Return the maximum value between ``x`` and ``y``. + Equivalent to :c:macro:`Py_LOCAL` but additionally requests the function + be inlined. - .. versionadded:: 3.3 +.. c:macro:: Py_LOCAL_SYMBOL -.. c:macro:: Py_MEMBER_SIZE(type, member) + Macro used to declare a symbol as local to the shared library (hidden). + On supported platforms, it ensures the symbol is not exported. - Return the size of a structure (``type``) ``member`` in bytes. + On compatible versions of GCC/Clang, it + expands to ``__attribute__((visibility("hidden")))``. - .. versionadded:: 3.6 +.. c:macro:: Py_EXPORTED_SYMBOL -.. c:macro:: Py_MIN(x, y) + Macro used to declare a symbol (function or data) as exported. + On Windows, this expands to ``__declspec(dllexport)``. + On compatible versions of GCC/Clang, it + expands to ``__attribute__((visibility("default")))``. + This macro is for defining the C API itself; extension modules should not use it. - Return the minimum value between ``x`` and ``y``. - .. versionadded:: 3.3 +.. c:macro:: Py_IMPORTED_SYMBOL -.. c:macro:: Py_NO_INLINE + Macro used to declare a symbol as imported. + On Windows, this expands to ``__declspec(dllimport)``. + This macro is for defining the C API itself; extension modules should not use it. - Disable inlining on a function. For example, it reduces the C stack - consumption: useful on LTO+PGO builds which heavily inline code (see - :issue:`33720`). - Usage:: +.. c:macro:: PyAPI_FUNC(type) - Py_NO_INLINE static int random(void) { return 4; } + Macro used by CPython to declare a function as part of the C API. + Its expansion depends on the platform and build configuration. + This macro is intended for defining CPython's C API itself; + extension modules should not use it for their own symbols. - .. versionadded:: 3.11 -.. c:macro:: Py_STRINGIFY(x) +.. c:macro:: PyAPI_DATA(type) - Convert ``x`` to a C string. E.g. ``Py_STRINGIFY(123)`` returns - ``"123"``. + Macro used by CPython to declare a public global variable as part of the C API. + Its expansion depends on the platform and build configuration. + This macro is intended for defining CPython's C API itself; + extension modules should not use it for their own symbols. - .. versionadded:: 3.4 -.. c:macro:: Py_UNREACHABLE() +Outdated macros +--------------- - Use this when you have a code path that cannot be reached by design. - For example, in the ``default:`` clause in a ``switch`` statement for which - all possible values are covered in ``case`` statements. Use this in places - where you might be tempted to put an ``assert(0)`` or ``abort()`` call. +The following :term:`soft deprecated` macros have been used to features that +have been standardized in C11 (or previous standards). - In release mode, the macro helps the compiler to optimize the code, and - avoids a warning about unreachable code. For example, the macro is - implemented with ``__builtin_unreachable()`` on GCC in release mode. +.. c:macro:: Py_ALIGNED(num) - A use for ``Py_UNREACHABLE()`` is following a call a function that - never returns but that is not declared :c:macro:`_Py_NO_RETURN`. + On some GCC-like compilers, specify alignment to *num* bytes. + This does nothing on other compilers. - If a code path is very unlikely code but can be reached under exceptional - case, this macro must not be used. For example, under low memory condition - or if a system call returns a value out of the expected range. In this - case, it's better to report the error to the caller. If the error cannot - be reported to caller, :c:func:`Py_FatalError` can be used. + Use the standard ``alignas`` specifier rather than this macro. - .. versionadded:: 3.7 + .. soft-deprecated:: 3.15 -.. c:macro:: Py_UNUSED(arg) +.. c:macro:: PY_FORMAT_SIZE_T - Use this for unused arguments in a function definition to silence compiler - warnings. Example: ``int func(int a, int Py_UNUSED(b)) { return a; }``. + The :c:func:`printf` formatting modifier for :c:type:`size_t`. + Use ``"z"`` directly instead. - .. versionadded:: 3.4 + .. soft-deprecated:: 3.15 -.. c:macro:: PyDoc_STRVAR(name, str) +.. c:macro:: Py_LL(number) + Py_ULL(number) - Creates a variable with name ``name`` that can be used in docstrings. - If Python is built without docstrings, the value will be empty. + Use *number* as a ``long long`` or ``unsigned long long`` integer literal, + respectively. - Use :c:macro:`PyDoc_STRVAR` for docstrings to support building - Python without docstrings, as specified in :pep:`7`. + Expands to *number* followed by ``LL`` or ``LLU``, respectively, but will + expand to some compiler-specific suffixes on some older compilers. - Example:: + Consider using the C99 standard suffixes ``LL`` and ``LLU`` directly. - PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element."); + .. soft-deprecated:: 3.15 - static PyMethodDef deque_methods[] = { - // ... - {"pop", (PyCFunction)deque_pop, METH_NOARGS, pop_doc}, - // ... - } +.. c:macro:: PY_LONG_LONG + PY_INT32_T + PY_UINT32_T + PY_INT64_T + PY_UINT64_T -.. c:macro:: PyDoc_STR(str) + Aliases for the types :c:type:`!long long`, :c:type:`!int32_t`, + :c:type:`!uint32_t`. :c:type:`!int64_t` and :c:type:`!uint64_t`, + respectively. + Historically, these types needed compiler-specific extensions. - Creates a docstring for the given input string or an empty string - if docstrings are disabled. + .. soft-deprecated:: 3.15 - Use :c:macro:`PyDoc_STR` in specifying docstrings to support - building Python without docstrings, as specified in :pep:`7`. +.. c:macro:: PY_LLONG_MIN + PY_LLONG_MAX + PY_ULLONG_MAX + PY_SIZE_MAX - Example:: + Aliases for the values :c:macro:`!LLONG_MIN`, :c:macro:`!LLONG_MAX`, + :c:macro:`!ULLONG_MAX`, and :c:macro:`!SIZE_MAX`, respectively. + Use these standard names instead. - static PyMethodDef pysqlite_row_methods[] = { - {"keys", (PyCFunction)pysqlite_row_keys, METH_NOARGS, - PyDoc_STR("Returns the keys of the row.")}, - {NULL, NULL} - }; + The required header, ````, + :ref:`is included ` in ``Python.h``. + + .. soft-deprecated:: 3.15 + +.. c:macro:: Py_MEMCPY(dest, src, n) + + This is an alias to :c:func:`!memcpy`. + + .. soft-deprecated:: 3.14 + Use :c:func:`!memcpy` directly instead. + +.. c:macro:: Py_UNICODE_SIZE + + Size of the :c:type:`!wchar_t` type. + Use ``sizeof(wchar_t)`` or ``WCHAR_WIDTH/8`` instead. + + The required header for the latter, ````, + :ref:`is included ` in ``Python.h``. + + .. soft-deprecated:: 3.15 + +.. c:macro:: Py_UNICODE_WIDE + + Defined if ``wchar_t`` can hold a Unicode character (UCS-4). + Use ``sizeof(wchar_t) >= 4`` instead + + .. soft-deprecated:: 3.15 + +.. c:macro:: Py_VA_COPY + + This is an alias to the C99-standard ``va_copy`` function. + + Historically, this would use a compiler-specific method to copy a ``va_list``. + + .. versionchanged:: 3.6 + This is now an alias to ``va_copy``. + + .. soft-deprecated:: 3.15 .. _api-objects: diff --git a/Doc/c-api/iter.rst b/Doc/c-api/iter.rst index bf9df62c6f17065..6cfd24c5ae60c80 100644 --- a/Doc/c-api/iter.rst +++ b/Doc/c-api/iter.rst @@ -54,6 +54,6 @@ There are two functions specifically for working with iterators. - ``PYGEN_RETURN`` if iterator returns. Return value is returned via *presult*. - ``PYGEN_NEXT`` if iterator yields. Yielded value is returned via *presult*. - - ``PYGEN_ERROR`` if iterator has raised and exception. *presult* is set to ``NULL``. + - ``PYGEN_ERROR`` if iterator has raised an exception. *presult* is set to ``NULL``. .. versionadded:: 3.10 diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 6b7ba8c99791634..bfbfe3c92799809 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -50,3 +50,72 @@ sentinel value is returned. callable object that can be called with no parameters; each call to it should return the next item in the iteration. When *callable* returns a value equal to *sentinel*, the iteration will be terminated. + + +Range Objects +^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyRange_Type + + The type object for :class:`range` objects. + + +.. c:function:: int PyRange_Check(PyObject *o) + + Return true if the object *o* is an instance of a :class:`range` object. + This function always succeeds. + + +Builtin Iterator Types +^^^^^^^^^^^^^^^^^^^^^^ + +These are built-in iteration types that are included in Python's C API, but +provide no additional functions. They are here for completeness. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * C type + * Python type + * * .. c:var:: PyTypeObject PyEnum_Type + * :py:class:`enumerate` + * * .. c:var:: PyTypeObject PyFilter_Type + * :py:class:`filter` + * * .. c:var:: PyTypeObject PyMap_Type + * :py:class:`map` + * * .. c:var:: PyTypeObject PyReversed_Type + * :py:class:`reversed` + * * .. c:var:: PyTypeObject PyZip_Type + * :py:class:`zip` + + +Other Iterator Objects +^^^^^^^^^^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyByteArrayIter_Type +.. c:var:: PyTypeObject PyBytesIter_Type +.. c:var:: PyTypeObject PyListIter_Type +.. c:var:: PyTypeObject PyListRevIter_Type +.. c:var:: PyTypeObject PySetIter_Type +.. c:var:: PyTypeObject PyTupleIter_Type +.. c:var:: PyTypeObject PyRangeIter_Type +.. c:var:: PyTypeObject PyLongRangeIter_Type +.. c:var:: PyTypeObject PyDictIterKey_Type +.. c:var:: PyTypeObject PyDictRevIterKey_Type +.. c:var:: PyTypeObject PyDictIterValue_Type +.. c:var:: PyTypeObject PyDictRevIterValue_Type +.. c:var:: PyTypeObject PyDictIterItem_Type +.. c:var:: PyTypeObject PyDictRevIterItem_Type +.. c:var:: PyTypeObject PyODictIter_Type + + Type objects for iterators of various built-in objects. + + Do not create instances of these directly; prefer calling + :c:func:`PyObject_GetIter` instead. + + Note that there is no guarantee that a given built-in type uses a given iterator + type. For example, iterating over :class:`range` will use one of two iterator + types depending on the size of the range. Other types may start using a + similar scheme in the future, without warning. diff --git a/Doc/c-api/lifecycle.rst b/Doc/c-api/lifecycle.rst index 5a170862a26f440..531c4080a0131c3 100644 --- a/Doc/c-api/lifecycle.rst +++ b/Doc/c-api/lifecycle.rst @@ -256,6 +256,8 @@ To allocate and free memory, see :ref:`allocating-objects`. collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set); this may change in the future. + .. versionadded:: 3.4 + .. c:function:: int PyObject_CallFinalizerFromDealloc(PyObject *op) @@ -266,6 +268,8 @@ To allocate and free memory, see :ref:`allocating-objects`. should happen. Otherwise, this function returns 0 and destruction can continue normally. + .. versionadded:: 3.4 + .. seealso:: :c:member:`~PyTypeObject.tp_dealloc` for example code. diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 758415a76e5cb41..8f560699d355e49 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -74,11 +74,25 @@ List Objects Like :c:func:`PyList_GetItemRef`, but returns a :term:`borrowed reference` instead of a :term:`strong reference`. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the list concurrently. Prefer :c:func:`PyList_GetItemRef`, which returns + a :term:`strong reference`. + .. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i) Similar to :c:func:`PyList_GetItem`, but without error checking. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the list concurrently. Prefer :c:func:`PyList_GetItemRef`, which returns + a :term:`strong reference`. + .. c:function:: int PyList_SetItem(PyObject *list, Py_ssize_t index, PyObject *item) @@ -108,6 +122,14 @@ List Objects is being replaced; any reference in *list* at position *i* will be leaked. + .. note:: + + In the :term:`free-threaded build`, this macro has no internal + synchronization. It is normally only used to fill in new lists where no + other thread has a reference to the list. If the list may be shared, + use :c:func:`PyList_SetItem` instead, which uses a :term:`per-object + lock`. + .. c:function:: int PyList_Insert(PyObject *list, Py_ssize_t index, PyObject *item) @@ -138,6 +160,12 @@ List Objects Return ``0`` on success, ``-1`` on failure. Indexing from the end of the list is not supported. + .. note:: + + In the :term:`free-threaded build`, when *itemlist* is a :class:`list`, + both *list* and *itemlist* are locked for the duration of the operation. + For other iterables (or ``NULL``), only *list* is locked. + .. c:function:: int PyList_Extend(PyObject *list, PyObject *iterable) @@ -150,6 +178,14 @@ List Objects .. versionadded:: 3.13 + .. note:: + + In the :term:`free-threaded build`, when *iterable* is a :class:`list`, + :class:`set`, :class:`dict`, or dict view, both *list* and *iterable* + (or its underlying dict) are locked for the duration of the operation. + For other iterables, only *list* is locked; *iterable* may be + concurrently modified by another thread. + .. c:function:: int PyList_Clear(PyObject *list) @@ -168,6 +204,14 @@ List Objects Sort the items of *list* in place. Return ``0`` on success, ``-1`` on failure. This is equivalent to ``list.sort()``. + .. note:: + + In the :term:`free-threaded build`, element comparison via + :meth:`~object.__lt__` can execute arbitrary Python code, during which + the :term:`per-object lock` may be temporarily released. For built-in + types (:class:`str`, :class:`int`, :class:`float`), the lock is not + released during comparison. + .. c:function:: int PyList_Reverse(PyObject *list) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 2d0bda76697e81c..874e422d4701dd8 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -40,9 +40,11 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Return a new :c:type:`PyLongObject` object from *v*, or ``NULL`` on failure. - The current implementation keeps an array of integer objects for all integers - between ``-5`` and ``256``. When you create an int in that range you actually - just get back a reference to the existing object. + .. impl-detail:: + + CPython keeps an array of integer objects for all integers + between ``-5`` and ``1024``. When you create an int in that range + you actually just get back a reference to the existing object. .. c:function:: PyObject* PyLong_FromUnsignedLong(unsigned long v) @@ -69,6 +71,12 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. on failure. +.. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) + + Return a new :c:type:`PyLongObject` object from a C :c:expr:`unsigned long long`, + or ``NULL`` on failure. + + .. c:function:: PyObject* PyLong_FromInt32(int32_t value) PyObject* PyLong_FromInt64(int64_t value) @@ -79,12 +87,6 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.14 -.. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) - - Return a new :c:type:`PyLongObject` object from a C :c:expr:`unsigned long long`, - or ``NULL`` on failure. - - .. c:function:: PyObject* PyLong_FromUInt32(uint32_t value) PyObject* PyLong_FromUInt64(uint64_t value) @@ -159,6 +161,17 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 +.. c:macro:: PyLong_FromPid(pid) + + Macro for creating a Python integer from a process identifier. + + This can be defined as an alias to :c:func:`PyLong_FromLong` or + :c:func:`PyLong_FromLongLong`, depending on the size of the system's + PID type. + + .. versionadded:: 3.2 + + .. c:function:: long PyLong_AsLong(PyObject *obj) .. index:: @@ -184,12 +197,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: long PyLong_AS_LONG(PyObject *obj) - A :term:`soft deprecated` alias. Exactly equivalent to the preferred ``PyLong_AsLong``. In particular, it can fail with :exc:`OverflowError` or another exception. - .. deprecated:: 3.14 - The function is soft deprecated. + .. soft-deprecated:: 3.14 .. c:function:: int PyLong_AsInt(PyObject *obj) @@ -372,6 +383,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Set *\*value* to a signed C :c:expr:`int32_t` or :c:expr:`int64_t` representation of *obj*. + If *obj* is not an instance of :c:type:`PyLongObject`, first call its + :meth:`~object.__index__` method (if present) to convert it to a + :c:type:`PyLongObject`. + If the *obj* value is out of range, raise an :exc:`OverflowError`. Set *\*value* and return ``0`` on success. @@ -436,8 +451,8 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Otherwise, returns the number of bytes required to store the value. If this is equal to or less than *n_bytes*, the entire value was copied. - All *n_bytes* of the buffer are written: large buffers are padded with - zeroes. + All *n_bytes* of the buffer are written: remaining bytes filled by + copies of the sign bit. If the returned value is greater than *n_bytes*, the value was truncated: as many of the lowest bits of the value as could fit are written, @@ -569,6 +584,17 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 +.. c:macro:: PyLong_AsPid(pid) + + Macro for converting a Python integer into a process identifier. + + This can be defined as an alias to :c:func:`PyLong_AsLong`, + :c:func:`PyLong_FromLongLong`, or :c:func:`PyLong_AsInt`, depending on the + size of the system's PID type. + + .. versionadded:: 3.2 + + .. c:function:: int PyLong_GetSign(PyObject *obj, int *sign) Get the sign of the integer object *obj*. @@ -659,7 +685,7 @@ Export API .. versionadded:: 3.14 -.. c:struct:: PyLongLayout +.. c:type:: PyLongLayout Layout of an array of "digits" ("limbs" in the GMP terminology), used to represent absolute value for arbitrary precision integers. @@ -699,7 +725,7 @@ Export API Get the native layout of Python :class:`int` objects. - See the :c:struct:`PyLongLayout` structure. + See the :c:type:`PyLongLayout` structure. The function must not be called before Python initialization nor after Python finalization. The returned layout is valid until Python is @@ -707,7 +733,7 @@ Export API in a process, and so it can be cached. -.. c:struct:: PyLongExport +.. c:type:: PyLongExport Export of a Python :class:`int` object. @@ -741,7 +767,7 @@ Export API Export a Python :class:`int` object. - *export_long* must point to a :c:struct:`PyLongExport` structure allocated + *export_long* must point to a :c:type:`PyLongExport` structure allocated by the caller. It must not be ``NULL``. On success, fill in *\*export_long* and return ``0``. @@ -771,7 +797,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer. .. versionadded:: 3.14 -.. c:struct:: PyLongWriter +.. c:type:: PyLongWriter A Python :class:`int` writer instance. @@ -799,7 +825,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer. The layout of *digits* is described by :c:func:`PyLong_GetNativeLayout`. Digits must be in the range [``0``; ``(1 << bits_per_digit) - 1``] - (where the :c:struct:`~PyLongLayout.bits_per_digit` is the number of bits + (where the :c:type:`~PyLongLayout.bits_per_digit` is the number of bits per digit). Any unused most significant digits must be set to ``0``. @@ -827,3 +853,31 @@ The :c:type:`PyLongWriter` API can be used to import an integer. If *writer* is ``NULL``, no operation is performed. The writer instance and the *digits* array are invalid after the call. + + +Deprecated API +^^^^^^^^^^^^^^ + +These macros are :term:`soft deprecated`. They describe parameters +of the internal representation of :c:type:`PyLongObject` instances. + +Use :c:func:`PyLong_GetNativeLayout` instead, along with :c:func:`PyLong_Export` +to read integer data or :c:type:`PyLongWriter` to write it. +These currently use the same layout, but are designed to continue working correctly +even if CPython's internal integer representation changes. + + +.. c:macro:: PyLong_SHIFT + + This is equivalent to :c:member:`~PyLongLayout.bits_per_digit` in + the output of :c:func:`PyLong_GetNativeLayout`. + + +.. c:macro:: PyLong_BASE + + This is currently equivalent to :c:expr:`1 << PyLong_SHIFT`. + + +.. c:macro:: PyLong_MASK + + This is currently equivalent to :c:expr:`(1 << PyLong_SHIFT) - 1` diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst index 1f55c0aa955c75b..2476ebb9b69dce0 100644 --- a/Doc/c-api/mapping.rst +++ b/Doc/c-api/mapping.rst @@ -102,7 +102,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. note:: - Exceptions which occur when this calls :meth:`~object.__getitem__` + Exceptions which occur when this calls the :meth:`~object.__getitem__` method are silently ignored. For proper error handling, use :c:func:`PyMapping_HasKeyWithError`, :c:func:`PyMapping_GetOptionalItem` or :c:func:`PyObject_GetItem()` instead. @@ -116,7 +116,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. note:: - Exceptions that occur when this calls :meth:`~object.__getitem__` + Exceptions that occur when this calls the :meth:`~object.__getitem__` method or while creating the temporary :class:`str` object are silently ignored. For proper error handling, use :c:func:`PyMapping_HasKeyStringWithError`, diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index 61218a1bf6f1716..668a163b2df5a11 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -82,7 +82,7 @@ The following functions allow marshalled values to be read back in. assumes that no further objects will be read from the file, allowing it to aggressively load file data into memory so that the de-serialization can operate from data in memory rather than reading a byte at a time from the - file. Only use these variant if you are certain that you won't be reading + file. Only use this variant if you are certain that you won't be reading anything else from the file. On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index df1bb0ce370919e..73310670ac371c9 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -7,10 +7,6 @@ Memory Management ***************** -.. sectionauthor:: Vladimir Marangozov - - - .. _memoryoverview: Overview @@ -81,7 +77,7 @@ memory footprint as a whole. Consequently, under certain circumstances, the Python memory manager may or may not trigger appropriate actions, like garbage collection, memory compaction or other preventive procedures. Note that by using the C library allocator as shown in the previous example, the allocated memory -for the I/O buffer escapes completely the Python memory manager. +for the I/O buffer completely escapes the Python memory manager. .. seealso:: @@ -102,7 +98,7 @@ All allocating functions belong to one of three different "domains" (see also strategies and are optimized for different purposes. The specific details on how every domain allocates memory or what internal functions each domain calls is considered an implementation detail, but for debugging purposes a simplified -table can be found at :ref:`here `. +table can be found at :ref:`default-memory-allocators`. The APIs used to allocate and free a block of memory must be from the same domain. For example, :c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. @@ -161,7 +157,7 @@ zero bytes. .. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize) - Allocates *nelem* elements each whose size in bytes is *elsize* and returns + Allocates *nelem* elements each of size *elsize* bytes and returns a pointer of type :c:expr:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. @@ -208,8 +204,11 @@ The following function sets, modeled after the ANSI C standard, but specifying behavior when requesting zero bytes, are available for allocating and releasing memory from the Python heap. -The :ref:`default memory allocator ` uses the -:ref:`pymalloc memory allocator `. +In the GIL-enabled build (default build) the +:ref:`default memory allocator ` uses the +:ref:`pymalloc memory allocator `, whereas in the +:term:`free-threaded build`, the default is the +:ref:`mimalloc memory allocator ` instead. .. warning:: @@ -219,6 +218,11 @@ The :ref:`default memory allocator ` uses the The default allocator is now pymalloc instead of system :c:func:`malloc`. +.. versionchanged:: 3.13 + + In the :term:`free-threaded ` build, the default allocator + is now :ref:`mimalloc `. + .. c:function:: void* PyMem_Malloc(size_t n) Allocates *n* bytes and returns a pointer of type :c:expr:`void*` to the @@ -231,7 +235,7 @@ The :ref:`default memory allocator ` uses the .. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize) - Allocates *nelem* elements each whose size in bytes is *elsize* and returns + Allocates *nelem* elements each of size *elsize* bytes and returns a pointer of type :c:expr:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. @@ -293,17 +297,39 @@ The following type-oriented macros are provided for convenience. Note that Same as :c:func:`PyMem_Free`. -In addition, the following macro sets are provided for calling the Python memory -allocator directly, without involving the C API functions listed above. However, -note that their use does not preserve binary compatibility across Python -versions and is therefore deprecated in extension modules. -* ``PyMem_MALLOC(size)`` -* ``PyMem_NEW(type, size)`` -* ``PyMem_REALLOC(ptr, size)`` -* ``PyMem_RESIZE(ptr, type, size)`` -* ``PyMem_FREE(ptr)`` -* ``PyMem_DEL(ptr)`` +Deprecated aliases +------------------ + +These are :term:`soft deprecated` aliases to existing functions and macros. +They exist solely for backwards compatibility. + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * Deprecated alias + * Corresponding function or macro + * * .. c:macro:: PyMem_MALLOC(size) + * :c:func:`PyMem_Malloc` + * * .. c:macro:: PyMem_NEW(type, size) + * :c:macro:`PyMem_New` + * * .. c:macro:: PyMem_REALLOC(ptr, size) + * :c:func:`PyMem_Realloc` + * * .. c:macro:: PyMem_RESIZE(ptr, type, size) + * :c:macro:`PyMem_Resize` + * * .. c:macro:: PyMem_FREE(ptr) + * :c:func:`PyMem_Free` + * * .. c:macro:: PyMem_DEL(ptr) + * :c:func:`PyMem_Free` + +.. versionchanged:: 3.4 + + The macros are now aliases of the corresponding functions and macros. + Previously, their behavior was the same, but their use did not necessarily + preserve binary compatibility across Python versions. + +.. deprecated:: 2.0 .. _objectinterface: @@ -322,7 +348,9 @@ memory from the Python heap. the :ref:`Customize Memory Allocators ` section. The :ref:`default object allocator ` uses the -:ref:`pymalloc memory allocator `. +:ref:`pymalloc memory allocator `. In the +:term:`free-threaded ` build, the default is the +:ref:`mimalloc memory allocator ` instead. .. warning:: @@ -340,7 +368,7 @@ The :ref:`default object allocator ` uses the .. c:function:: void* PyObject_Calloc(size_t nelem, size_t elsize) - Allocates *nelem* elements each whose size in bytes is *elsize* and returns + Allocates *nelem* elements each of size *elsize* bytes and returns a pointer of type :c:expr:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. @@ -402,14 +430,16 @@ Default Memory Allocators Default memory allocators: -=============================== ==================== ================== ===================== ==================== -Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc -=============================== ==================== ================== ===================== ==================== -Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` -Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug -Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` -Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug -=============================== ==================== ================== ===================== ==================== +=================================== ======================= ==================== ====================== ====================== +Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc +=================================== ======================= ==================== ====================== ====================== +Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` +Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug +Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` +Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug +Free-threaded build ``"mimalloc"`` ``mimalloc`` ``mimalloc`` ``mimalloc`` +Free-threaded debug build ``"mimalloc_debug"`` ``mimalloc`` + debug ``mimalloc`` + debug ``mimalloc`` + debug +=================================== ======================= ==================== ====================== ====================== Legend: @@ -417,8 +447,7 @@ Legend: * ``malloc``: system allocators from the standard C library, C functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. * ``pymalloc``: :ref:`pymalloc memory allocator `. -* ``mimalloc``: :ref:`mimalloc memory allocator `. The pymalloc - allocator will be used if mimalloc support isn't available. +* ``mimalloc``: :ref:`mimalloc memory allocator `. * "+ debug": with :ref:`debug hooks on the Python memory allocators `. * "Debug build": :ref:`Python build in debug mode `. @@ -655,7 +684,11 @@ The pymalloc allocator Python has a *pymalloc* allocator optimized for small objects (smaller or equal to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" with a fixed size of either 256 KiB on 32-bit platforms or 1 MiB on 64-bit -platforms. It falls back to :c:func:`PyMem_RawMalloc` and +platforms. When Python is configured with :option:`--with-pymalloc-hugepages`, +the arena size on 64-bit platforms is increased to 2 MiB to match the huge page +size, and arena allocation will attempt to use huge pages (``MAP_HUGETLB`` on +Linux, ``MEM_LARGE_PAGES`` on Windows) with automatic fallback to regular pages. +It falls back to :c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. *pymalloc* is the :ref:`default allocator ` of the @@ -711,9 +744,27 @@ The mimalloc allocator .. versionadded:: 3.13 -Python supports the mimalloc allocator when the underlying platform support is available. -mimalloc "is a general purpose allocator with excellent performance characteristics. -Initially developed by Daan Leijen for the runtime systems of the Koka and Lean languages." +Python supports the `mimalloc `__ +allocator when the underlying platform support is available. +mimalloc is a general purpose allocator with excellent performance +characteristics, initially developed by Daan Leijen for the runtime systems +of the Koka and Lean languages. + +Unlike :ref:`pymalloc `, which is optimized for small objects (512 +bytes or fewer), mimalloc handles allocations of any size. + +In the :term:`free-threaded ` build, mimalloc is the default +and **required** allocator for the :c:macro:`PYMEM_DOMAIN_MEM` and +:c:macro:`PYMEM_DOMAIN_OBJ` domains. It cannot be disabled in free-threaded +builds. The free-threaded build uses per-thread mimalloc heaps, which allows +allocation and deallocation to proceed without locking in most cases. + +In the default (non-free-threaded) build, mimalloc is available but not the +default allocator. It can be selected at runtime using +:envvar:`PYTHONMALLOC`\ ``=mimalloc`` (or ``mimalloc_debug`` to include +:ref:`debug hooks `). It can be disabled at build time +using the :option:`--without-mimalloc` configure option, but this option +cannot be combined with :option:`--disable-gil`. tracemalloc C API ================= diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index f6038032805259f..e4ac8b57673407c 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -13,6 +13,12 @@ A :class:`memoryview` object exposes the C level :ref:`buffer interface any other object. +.. c:var:: PyTypeObject PyMemoryView_Type + + This instance of :c:type:`PyTypeObject` represents the Python memoryview + type. This is the same object as :class:`memoryview` in the Python layer. + + .. c:function:: PyObject *PyMemoryView_FromObject(PyObject *obj) Create a memoryview object from an object that provides the buffer interface. diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index c8edcecc5b419f3..9f68abba66bc5d6 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -3,17 +3,16 @@ .. _moduleobjects: Module Objects --------------- +============== .. index:: pair: object; module - .. c:var:: PyTypeObject PyModule_Type .. index:: single: ModuleType (in module types) This instance of :c:type:`PyTypeObject` represents the Python module type. This - is exposed to Python programs as ``types.ModuleType``. + is exposed to Python programs as :py:class:`types.ModuleType`. .. c:function:: int PyModule_Check(PyObject *p) @@ -71,6 +70,9 @@ Module Objects ``PyObject_*`` functions rather than directly manipulate a module's :attr:`~object.__dict__`. + The returned reference is borrowed from the module; it is valid until + the module is destroyed. + .. c:function:: PyObject* PyModule_GetNameObject(PyObject *module) @@ -90,18 +92,19 @@ Module Objects Similar to :c:func:`PyModule_GetNameObject` but return the name encoded to ``'utf-8'``. -.. c:function:: void* PyModule_GetState(PyObject *module) - - Return the "state" of the module, that is, a pointer to the block of memory - allocated at module creation time, or ``NULL``. See - :c:member:`PyModuleDef.m_size`. - + The returned buffer is only valid until the module is renamed or destroyed. + Note that Python code may rename a module by setting its :py:attr:`~module.__name__` + attribute. .. c:function:: PyModuleDef* PyModule_GetDef(PyObject *module) Return a pointer to the :c:type:`PyModuleDef` struct from which the module was created, or ``NULL`` if the module wasn't created from a definition. + On error, return ``NULL`` with an exception set. + Use :c:func:`PyErr_Occurred` to tell this case apart from a missing + :c:type:`!PyModuleDef`. + .. c:function:: PyObject* PyModule_GetFilenameObject(PyObject *module) @@ -122,185 +125,200 @@ Module Objects Similar to :c:func:`PyModule_GetFilenameObject` but return the filename encoded to 'utf-8'. + The returned buffer is only valid until the module's :py:attr:`~module.__file__` attribute + is reassigned or the module is destroyed. + .. deprecated:: 3.2 :c:func:`PyModule_GetFilename` raises :exc:`UnicodeEncodeError` on unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. -.. _pymoduledef: +.. _c_module_slots: +.. _pymoduledef_slot: -Module definitions ------------------- +Module definition +----------------- -The functions in the previous section work on any module object, including -modules imported from Python code. +Modules created using the C API are typically defined using an +array of :c:type:`PySlot` structs, which provides a "description" of how a +module should be created. +See :ref:`capi-slots` for more information on slots in general. -Modules defined using the C API typically use a *module definition*, -:c:type:`PyModuleDef` -- a statically allocated, constant “description" of -how a module should be created. +.. versionchanged:: 3.15 -The definition is usually used to define an extension's “main” module object -(see :ref:`extension-modules` for details). -It is also used to -:ref:`create extension modules dynamically `. + Previously, a :c:type:`PyModuleDef` struct was necessary to define modules. + The older way of defining modules is still available: consult either the + :ref:`pymoduledef` section or earlier versions of this documentation + if you plan to support earlier Python versions. -Unlike :c:func:`PyModule_New`, the definition allows management of -*module state* -- a piece of memory that is allocated and cleared together -with the module object. -Unlike the module's Python attributes, Python code cannot replace or delete -data stored in module state. +The slots array is usually used to define an extension module's “main” +module object (see :ref:`extension-modules` for details). +It can also be used to +:ref:`create extension modules dynamically `. -.. c:type:: PyModuleDef +Unless specified otherwise, the same slot ID may not be repeated +in an array of slots. - The module definition struct, which holds all information needed to create - a module object. - This structure must be statically allocated (or be otherwise guaranteed - to be valid while any modules created from it exist). - Usually, there is only one variable of this type for each extension module. - .. c:member:: PyModuleDef_Base m_base +Metadata slots +.............. - Always initialize this member to :c:macro:`PyModuleDef_HEAD_INIT`. +.. c:macro:: Py_mod_name - .. c:member:: const char *m_name + :c:member:`Slot ID ` for the name of the new module, + as a NUL-terminated UTF8-encoded ``const char *``. - Name for the new module. + Note that modules are typically created using a + :py:class:`~importlib.machinery.ModuleSpec`, and when they are, the + name from the spec will be used instead of :c:data:`!Py_mod_name`. + However, it is still recommended to include this slot for introspection + and debugging purposes. - .. c:member:: const char *m_doc + .. versionadded:: 3.15 - Docstring for the module; usually a docstring variable created with - :c:macro:`PyDoc_STRVAR` is used. + Use :c:member:`PyModuleDef.m_name` instead to support previous versions. - .. c:member:: Py_ssize_t m_size +.. c:macro:: Py_mod_doc - Module state may be kept in a per-module memory area that can be - retrieved with :c:func:`PyModule_GetState`, rather than in static globals. - This makes modules safe for use in multiple sub-interpreters. + :c:type:`Slot ID ` for the docstring of the new + module, as a NUL-terminated UTF8-encoded ``const char *``. - This memory area is allocated based on *m_size* on module creation, - and freed when the module object is deallocated, after the - :c:member:`~PyModuleDef.m_free` function has been called, if present. + Usually it is set to a variable created with :c:macro:`PyDoc_STRVAR`. - Setting it to a non-negative value means that the module can be - re-initialized and specifies the additional amount of memory it requires - for its state. + .. versionadded:: 3.15 - Setting ``m_size`` to ``-1`` means that the module does not support - sub-interpreters, because it has global state. - Negative ``m_size`` is only allowed when using - :ref:`legacy single-phase initialization ` - or when :ref:`creating modules dynamically `. + Use :c:member:`PyModuleDef.m_doc` instead to support previous versions. - See :PEP:`3121` for more details. - .. c:member:: PyMethodDef* m_methods +Feature slots +............. - A pointer to a table of module-level functions, described by - :c:type:`PyMethodDef` values. Can be ``NULL`` if no functions are present. +.. c:macro:: Py_mod_abi - .. c:member:: PyModuleDef_Slot* m_slots + :c:member:`Slot ID ` whose value points to + a :c:struct:`PyABIInfo` structure describing the ABI that + the extension is using. - An array of slot definitions for multi-phase initialization, terminated by - a ``{0, NULL}`` entry. - When using legacy single-phase initialization, *m_slots* must be ``NULL``. + A suitable :c:struct:`!PyABIInfo` variable can be defined using the + :c:macro:`PyABIInfo_VAR` macro, as in: - .. versionchanged:: 3.5 + .. code-block:: c - Prior to version 3.5, this member was always set to ``NULL``, - and was defined as: + PyABIInfo_VAR(abi_info); - .. c:member:: inquiry m_reload + static PySlot mymodule_slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + ... + }; - .. c:member:: traverseproc m_traverse + When creating a module, Python checks the value of this slot + using :c:func:`PyABIInfo_Check`. - A traversal function to call during GC traversal of the module object, or - ``NULL`` if not needed. + This slot is required, except for modules created from + :c:struct:`PyModuleDef`. - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. + .. versionadded:: 3.15 - .. versionchanged:: 3.9 - No longer called before the module state is allocated. +.. c:macro:: Py_mod_multiple_interpreters - .. c:member:: inquiry m_clear + :c:member:`Slot ID ` whose value is one of: - A clear function to call during GC clearing of the module object, or - ``NULL`` if not needed. + .. c:namespace:: NULL - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. + .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED - Like :c:member:`PyTypeObject.tp_clear`, this function is not *always* - called before a module is deallocated. For example, when reference - counting is enough to determine that an object is no longer used, - the cyclic garbage collector is not involved and - :c:member:`~PyModuleDef.m_free` is called directly. + The module does not support being imported in subinterpreters. - .. versionchanged:: 3.9 - No longer called before the module state is allocated. + .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED + + The module supports being imported in subinterpreters, + but only when they share the main interpreter's GIL. + (See :ref:`isolating-extensions-howto`.) - .. c:member:: freefunc m_free + .. c:macro:: Py_MOD_PER_INTERPRETER_GIL_SUPPORTED - A function to call during deallocation of the module object, or ``NULL`` - if not needed. + The module supports being imported in subinterpreters, + even when they have their own GIL. + (See :ref:`isolating-extensions-howto`.) - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. + This slot determines whether or not importing this module + in a subinterpreter will fail. - .. versionchanged:: 3.9 - No longer called before the module state is allocated. + If ``Py_mod_multiple_interpreters`` is not specified, the import + machinery defaults to ``Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED``. + For historical reasons, the values are declared as pointers (``void *``). + When using :c:type:`PySlot` arrays, use :c:macro:`PySlot_DATA` for + :c:macro:`!Py_mod_multiple_interpreters`: -Module slots -............ + .. code-block:: c -.. c:type:: PyModuleDef_Slot + PySlot_DATA(Py_mod_multiple_interpreters, + Py_MOD_PER_INTERPRETER_GIL_SUPPORTED) - .. c:member:: int slot + .. versionadded:: 3.12 - A slot ID, chosen from the available values explained below. +.. c:macro:: Py_mod_gil - .. c:member:: void* value + :c:member:`Slot ID ` whose value is one of: - Value of the slot, whose meaning depends on the slot ID. + .. c:namespace:: NULL - .. versionadded:: 3.5 + .. c:macro:: Py_MOD_GIL_USED + + The module depends on the presence of the global interpreter lock (GIL), + and may access global state without synchronization. + + .. c:macro:: Py_MOD_GIL_NOT_USED + + The module is safe to run without an active GIL. + + This slot is ignored by Python builds not configured with + :option:`--disable-gil`. Otherwise, it determines whether or not importing + this module will cause the GIL to be automatically enabled. See + :ref:`whatsnew313-free-threaded-cpython` for more detail. + + If ``Py_mod_gil`` is not specified, the import machinery defaults to + ``Py_MOD_GIL_USED``. -The available slot types are: + For historical reasons, the values are declared as pointers (``void *``). + When using :c:type:`PySlot` arrays, use :c:macro:`PySlot_DATA` for + :c:macro:`!Py_mod_gil`: + + .. code-block:: c + + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED) + + .. versionadded:: 3.13 + + +Creation and initialization slots +................................. .. c:macro:: Py_mod_create - Specifies a function that is called to create the module object itself. - The *value* pointer of this slot must point to a function of the signature: + :c:member:`Slot ID ` for a function that creates + the module object itself. + The function must have the signature: .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) :no-index-entry: :no-contents-entry: - The function receives a :py:class:`~importlib.machinery.ModuleSpec` - instance, as defined in :PEP:`451`, and the module definition. - It should return a new module object, or set an error + The function will be called with: + + - *spec*: a ``ModuleSpec``-like object, meaning that any attributes defined + for :py:class:`importlib.machinery.ModuleSpec` have matching semantics. + However, any of the attributes may be missing. + - *def*: ``NULL``, or the module definition if the module is created from one. + + The function should return a new module object, or set an error and return ``NULL``. This function should be kept minimal. In particular, it should not call arbitrary Python code, as trying to import the same module again may result in an infinite loop. - Multiple ``Py_mod_create`` slots may not be specified in one module - definition. - If ``Py_mod_create`` is not specified, the import machinery will create a normal module object using :c:func:`PyModule_New`. The name is taken from *spec*, not the definition, to allow extension modules to dynamically adjust @@ -308,96 +326,498 @@ The available slot types are: names through symlinks, all while sharing a single module definition. There is no requirement for the returned object to be an instance of - :c:type:`PyModule_Type`. Any type can be used, as long as it supports - setting and getting import-related attributes. - However, only ``PyModule_Type`` instances may be returned if the - ``PyModuleDef`` has non-``NULL`` ``m_traverse``, ``m_clear``, - ``m_free``; non-zero ``m_size``; or slots other than ``Py_mod_create``. + :c:type:`PyModule_Type`. + However, some slots may only be used with + :c:type:`!PyModule_Type` instances; in particular: + + - :c:macro:`Py_mod_exec`, + - :ref:`module state slots ` (``Py_mod_state_*``), + - :c:macro:`Py_mod_token`. + + .. versionadded:: 3.5 + + .. versionchanged:: 3.15 + + The *slots* argument may be a ``ModuleSpec``-like object, rather than + a true :py:class:`~importlib.machinery.ModuleSpec` instance. + Note that previous versions of CPython did not enforce this. + + The *def* argument may now be ``NULL``, since modules are not necessarily + made from definitions. .. c:macro:: Py_mod_exec - Specifies a function that is called to *execute* the module. - This is equivalent to executing the code of a Python module: typically, - this function adds classes and constants to the module. + :c:member:`Slot ID ` for a function that will + :dfn:`execute`, or initialize, the module. + This function does the equivalent to executing the code of a Python module: + typically, it adds classes and constants to the module. The signature of the function is: .. c:function:: int exec_module(PyObject* module) :no-index-entry: :no-contents-entry: - If multiple ``Py_mod_exec`` slots are specified, they are processed in the - order they appear in the *m_slots* array. + See the :ref:`capi-module-support-functions` section for some useful + functions to call. -.. c:macro:: Py_mod_multiple_interpreters + For backwards compatibility, the :c:type:`PyModuleDef.m_slots` array may + contain multiple :c:macro:`!Py_mod_exec` slots; these are processed in the + order they appear in the array. + Elsewhere (that is, in arguments to :c:func:`PyModule_FromSlotsAndSpec` + and in return values of :samp:`PyModExport_{}`), repeating the slot + is not allowed. - Specifies one of the following values: + .. versionadded:: 3.5 - .. c:namespace:: NULL + .. versionchanged:: 3.15 - .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED + Repeated ``Py_mod_exec`` slots are disallowed, except in + :c:type:`PyModuleDef.m_slots`. - The module does not support being imported in subinterpreters. +.. c:macro:: Py_mod_methods - .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED + :c:member:`Slot ID ` for a table of module-level + functions, as an array of :c:type:`PyMethodDef` values suitable as the + *functions* argument to :c:func:`PyModule_AddFunctions`. - The module supports being imported in subinterpreters, - but only when they share the main interpreter's GIL. - (See :ref:`isolating-extensions-howto`.) + Like other slot IDs, a slots array may only contain one + :c:macro:`!Py_mod_methods` entry. + To add functions from multiple :c:type:`PyMethodDef` arrays, call + :c:func:`PyModule_AddFunctions` in the :c:macro:`Py_mod_exec` function. - .. c:macro:: Py_MOD_PER_INTERPRETER_GIL_SUPPORTED + The table must be statically allocated (or otherwise guaranteed to outlive + the module object). - The module supports being imported in subinterpreters, - even when they have their own GIL. - (See :ref:`isolating-extensions-howto`.) + .. versionadded:: 3.15 - This slot determines whether or not importing this module - in a subinterpreter will fail. + Use :c:member:`PyModuleDef.m_methods` instead to support previous versions. - Multiple ``Py_mod_multiple_interpreters`` slots may not be specified - in one module definition. +.. _ext-module-state: - If ``Py_mod_multiple_interpreters`` is not specified, the import - machinery defaults to ``Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED``. +Module state +------------ - .. versionadded:: 3.12 +Extension modules can have *module state* -- a +piece of memory that is allocated on module creation, +and freed when the module object is deallocated. +The module state is specified using :ref:`dedicated slots `. -.. c:macro:: Py_mod_gil +A typical use of module state is storing an exception type -- or indeed *any* +type object defined by the module -- - Specifies one of the following values: +Unlike the module's Python attributes, Python code cannot replace or delete +data stored in module state. - .. c:namespace:: NULL +Keeping per-module information in attributes and module state, rather than in +static globals, makes module objects *isolated* and safer for use in +multiple sub-interpreters. +It also helps Python do an orderly clean-up when it shuts down. - .. c:macro:: Py_MOD_GIL_USED +Extensions that keep references to Python objects as part of module state must +implement :c:macro:`Py_mod_state_traverse` and :c:macro:`Py_mod_state_clear` +functions to avoid reference leaks. - The module depends on the presence of the global interpreter lock (GIL), - and may access global state without synchronization. +To retrieve the state from a given module, use the following functions: - .. c:macro:: Py_MOD_GIL_NOT_USED +.. c:function:: void* PyModule_GetState(PyObject *module) - The module is safe to run without an active GIL. + Return the "state" of the module, that is, a pointer to the block of memory + allocated at module creation time, or ``NULL``. See + :c:macro:`Py_mod_state_size`. - This slot is ignored by Python builds not configured with - :option:`--disable-gil`. Otherwise, it determines whether or not importing - this module will cause the GIL to be automatically enabled. See - :ref:`whatsnew313-free-threaded-cpython` for more detail. + On error, return ``NULL`` with an exception set. + Use :c:func:`PyErr_Occurred` to tell this case apart from missing + module state. - Multiple ``Py_mod_gil`` slots may not be specified in one module definition. - If ``Py_mod_gil`` is not specified, the import machinery defaults to - ``Py_MOD_GIL_USED``. +.. c:function:: int PyModule_GetStateSize(PyObject *module, Py_ssize_t *result) - .. versionadded:: 3.13 + Set *\*result* to the size of *module*'s state, as specified + using :c:macro:`Py_mod_state_size` (or :c:member:`PyModuleDef.m_size`), + and return 0. + On error, set *\*result* to -1, and return -1 with an exception set. -.. _moduledef-dynamic: + .. versionadded:: 3.15 + + + +.. _ext-module-state-slots: + +Slots for defining module state +............................... + +The following :c:member:`slot IDs ` are available for +defining the module state. + +.. c:macro:: Py_mod_state_size + + :c:member:`Slot ID ` for the size of the module state, + in bytes. + + Setting the value to a non-negative value means that the module can be + re-initialized and specifies the additional amount of memory it requires + for its state. + + See :PEP:`3121` for more details. + + Use :c:func:`PyModule_GetStateSize` to retrieve the size of a given module. + + .. versionadded:: 3.15 + + Use :c:member:`PyModuleDef.m_size` instead to support previous versions. + +.. c:macro:: Py_mod_state_traverse + + :c:member:`Slot ID ` for a traversal function to call + during GC traversal of the module object. + + The signature of the function, and meanings of the arguments, + is similar as for :c:member:`PyTypeObject.tp_traverse`: + + .. c:function:: int traverse_module_state(PyObject *module, visitproc visit, void *arg) + :no-index-entry: + :no-contents-entry: + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if the state size + (:c:data:`Py_mod_state_size`) is greater than 0 and the module state + (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + + .. versionadded:: 3.15 + + Use :c:member:`PyModuleDef.m_size` instead to support previous versions. + +.. c:macro:: Py_mod_state_clear + + :c:member:`Slot ID ` for a clear function to call + during GC clearing of the module object. + + The signature of the function is: + + .. c:function:: int clear_module_state(PyObject* module) + :no-index-entry: + :no-contents-entry: + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if the state size + (:c:data:`Py_mod_state_size`) is greater than 0 and the module state + (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + + Like :c:member:`PyTypeObject.tp_clear`, this function is not *always* + called before a module is deallocated. For example, when reference + counting is enough to determine that an object is no longer used, + the cyclic garbage collector is not involved and + the :c:macro:`Py_mod_state_free` function is called directly. + + .. versionadded:: 3.15 + + Use :c:member:`PyModuleDef.m_clear` instead to support previous versions. + +.. c:macro:: Py_mod_state_free + + :c:member:`Slot ID ` for a function to call during + deallocation of the module object. + + The signature of the function is: + + .. c:function:: int free_module_state(PyObject* module) + :no-index-entry: + :no-contents-entry: + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if the state size + (:c:data:`Py_mod_state_size`) is greater than 0 and the module state + (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + + .. versionadded:: 3.15 + + Use :c:member:`PyModuleDef.m_free` instead to support previous versions. + + +.. _ext-module-token: + +Module token +............ + +Each module may have an associated *token*: a pointer-sized value intended to +identify of the module state's memory layout. +This means that if you have a module object, but you are not sure if it +“belongs” to your extension, you can check using code like this: + +.. code-block:: c + + PyObject *module = + + void *module_token; + if (PyModule_GetToken(module, &module_token) < 0) { + return NULL; + } + if (module_token != your_token) { + PyErr_SetString(PyExc_ValueError, "unexpected module") + return NULL; + } + + // This module's state has the expected memory layout; it's safe to cast + struct my_state state = (struct my_state*)PyModule_GetState(module) + +A module's token -- and the *your_token* value to use in the above code -- is: + +- For modules created with :c:type:`PyModuleDef`: the address of that + :c:type:`PyModuleDef`; +- For modules defined with the :c:macro:`Py_mod_token` slot: the value + of that slot; +- For modules created from an ``PyModExport_*`` + :ref:`export hook `: the slots array that the export + hook returned (unless overridden with :c:macro:`Py_mod_token`). + +.. c:macro:: Py_mod_token + + :c:member:`Slot ID ` for the module token. + + If you use this slot to set the module token (rather than rely on the + default), you must ensure that: + + * The pointer outlives the class, so it's not reused for something else + while the class exists. + * It "belongs" to the extension module where the class lives, so it will not + clash with other extensions. + * If the token points to a :c:type:`PyModuleDef` struct, the module should + behave as if it was created from that :c:type:`PyModuleDef`. + In particular, the module state must have matching layout and semantics. + + Modules created from :c:type:`PyModuleDef` always use the address of + the :c:type:`PyModuleDef` as the token. + This means that :c:macro:`!Py_mod_token` cannot be used in + :c:member:`PyModuleDef.m_slots`. + + .. versionadded:: 3.15 + +.. c:function:: int PyModule_GetToken(PyObject *module, void** result) + + Set *\*result* to the module token for *module* and return 0. + + On error, set *\*result* to NULL, and return -1 with an exception set. + + .. versionadded:: 3.15 + +See also :c:func:`PyType_GetModuleByToken`. + + +.. _module-from-slots: Creating extension modules dynamically -------------------------------------- -The following functions may be used to create a module outside of an -extension's :ref:`initialization function `. -They are also used in -:ref:`single-phase initialization `. +The following functions may be used to create an extension module dynamically, +rather than from an extension's :ref:`export hook `. + +.. c:function:: PyObject *PyModule_FromSlotsAndSpec(const PySlot *slots, PyObject *spec) + + Create a new module object, given an array of :ref:`slots ` + and the :py:class:`~importlib.machinery.ModuleSpec` *spec*. + + The *slots* argument must point to an array of :c:type:`PySlot` + structures, terminated by an entry with slot ID of 0 + (typically written as :c:macro:`PySlot_END`). + The array must include a :c:data:`Py_mod_abi` entry. + + The *spec* argument may be any ``ModuleSpec``-like object, as described + in :c:macro:`Py_mod_create` documentation. + Currently, the *spec* must have a ``name`` attribute. + + On success, return the new module. + On error, return ``NULL`` with an exception set. + + Note that this does not process the module's execution slot + (:c:data:`Py_mod_exec`). + Both :c:func:`!PyModule_FromSlotsAndSpec` and :c:func:`PyModule_Exec` + must be called to fully initialize a module. + (See also :ref:`multi-phase-initialization`.) + + .. versionadded:: 3.15 + +.. c:function:: int PyModule_Exec(PyObject *module) + + Execute the :c:data:`Py_mod_exec` slot(s) of *module*. + + On success, return 0. + On error, return -1 with an exception set. + + For clarity: If *module* has no slots, for example if it uses + :ref:`legacy single-phase initialization `, + this function does nothing and returns 0. + + .. versionadded:: 3.15 + + + +.. _pymoduledef: + +Module definition struct +------------------------ + +Traditionally, extension modules were defined using a *module definition* +as the “description" of how a module should be created. +Rather than using an array of :ref:`slots ` directly, +the definition has dedicated members for most common functionality, +and allows additional slots as an extension mechanism. + +This way of defining modules is still available and there are no plans to +remove it. + +.. c:type:: PyModuleDef + + The module definition struct, which holds information needed to create + a module object. + + This structure must be statically allocated (or be otherwise guaranteed + to be valid while any modules created from it exist). + Usually, there is only one variable of this type for each extension module + defined this way. + + The struct, including all members, is part of the + :ref:`Stable ABI ` for non-free-threaded builds (``abi3``). + In the Stable ABI for free-threaded builds (``abi3t``), + this struct is opaque, and unusable in practice; see :ref:`pymoduledef_slot` + for a replacement. + + .. c:member:: PyModuleDef_Base m_base + + Always initialize this member to :c:macro:`PyModuleDef_HEAD_INIT`: + + .. c:namespace:: NULL + + .. c:type:: PyModuleDef_Base + + The type of :c:member:`!PyModuleDef.m_base`. + + The struct is part of the :ref:`Stable ABI ` for + non-free-threaded builds (``abi3``). + In the Stable ABI for Free-Threaded Builds + (``abi3t``), this struct is opaque, and unusable in practice. + + .. c:macro:: PyModuleDef_HEAD_INIT + + The required initial value for :c:member:`!PyModuleDef.m_base`. + + .. c:member:: const char *m_name + + Corresponds to the :c:macro:`Py_mod_name` slot. + + .. c:member:: const char *m_doc + + These members correspond to the :c:macro:`Py_mod_doc` slot. + Setting this to NULL is equivalent to omitting the slot. + + .. c:member:: Py_ssize_t m_size + + Corresponds to the :c:macro:`Py_mod_state_size` slot. + Setting this to zero is equivalent to omitting the slot. + + When using :ref:`legacy single-phase initialization ` + or when creating modules dynamically using :c:func:`PyModule_Create` + or :c:func:`PyModule_Create2`, :c:member:`!m_size` may be set to -1. + This indicates that the module does not support sub-interpreters, + because it has global state. + + .. c:member:: PyMethodDef *m_methods + + Corresponds to the :c:macro:`Py_mod_methods` slot. + Setting this to NULL is equivalent to omitting the slot. + + .. c:member:: PyModuleDef_Slot* m_slots + + An array of additional slots, terminated by a ``{0, NULL}`` entry. + Note that the entries use the older :c:type:`PyModuleDef_Slot` structure, + rather than :c:type:`PySlot`. + + If the array contains slots corresponding to :c:type:`PyModuleDef` + members, the values must match. + For example, if you use :c:macro:`Py_mod_name` in :c:member:`!m_slots`, + :c:member:`PyModuleDef.m_name` must be set to the same pointer + (not just an equal string). + + .. versionchanged:: 3.5 + + Prior to version 3.5, this member was always set to ``NULL``, + and was defined as: + + .. c:member:: inquiry m_reload + + .. c:namespace:: NULL + + .. c:type:: PyModuleDef_Slot + + Older structure defining additional slots of a module. + + Note that a :c:type:`!PyModuleDef_Slot` array may be included in a + :c:type:`!PySlot` array using :c:macro:`Py_mod_slots`, + and vice versa using :c:macro:`Py_slot_subslots`. + + Each :c:type:`!PyModuleDef_Slot` structure ``modslot`` is interpreted + as the following :c:type:`PySlot` structure:: + + (PySlot){ + .sl_id=modslot.slot, + .sl_flags=PySlot_INTPTR | sub_static, + .sl_ptr=modslot.value + } + + where ``sub_static`` is ``PySlot_STATIC`` if the slot requires + the flag (such as for :c:macro:`Py_mod_methods`), or if this flag + is present on the "parent" :c:macro:`!Py_mod_slots` slot (if any). + + .. c:member:: int slot + + Corresponds to :c:member:`PySlot.sl_id`. + + .. c:member:: void* value + + Corresponds to :c:member:`PySlot.sl_ptr`. + + .. versionadded:: 3.5 + + .. c:member:: traverseproc m_traverse + inquiry m_clear + freefunc m_free + + These members correspond to the :c:macro:`Py_mod_state_traverse`, + :c:macro:`Py_mod_state_clear`, and :c:macro:`Py_mod_state_free` slots, + respectively. + + Setting these members to NULL is equivalent to omitting the + corresponding slots. + + .. versionchanged:: 3.9 + + :c:member:`m_traverse`, :c:member:`m_clear` and :c:member:`m_free` + functions are no longer called before the module state is allocated. + + +.. c:var:: PyTypeObject PyModuleDef_Type + + The type of ``PyModuleDef`` objects. + + +.. c:macro:: Py_mod_slots + + :c:member:`Slot ID ` that works like + :c:macro:`Py_slot_subslots`, except it specifies an array of + :c:type:`PyModuleDef_Slot` structures. + + .. versionadded:: 3.15 + +.. _moduledef-dynamic: + +The following API can be used to create modules from a :c:type:`!PyModuleDef` +struct: .. c:function:: PyObject* PyModule_Create(PyModuleDef *def) @@ -433,6 +853,10 @@ They are also used in .. versionadded:: 3.5 + .. soft-deprecated:: next + + Prefer :c:func:`PyModule_FromSlotsAndSpec` in new code. + .. c:function:: PyObject * PyModule_FromDefAndSpec2(PyModuleDef *def, PyObject *spec, int module_api_version) Create a new module object, given the definition in *def* and the @@ -453,33 +877,47 @@ They are also used in .. versionadded:: 3.5 + .. soft-deprecated:: next + + Prefer :c:func:`PyModule_FromSlotsAndSpec` in new code. + .. c:function:: int PyModule_ExecDef(PyObject *module, PyModuleDef *def) Process any execution slots (:c:data:`Py_mod_exec`) given in *def*. .. versionadded:: 3.5 + .. soft-deprecated:: next + + To run a module's own execution slots, prefer :c:func:`PyModule_Exec`, + which works on modules that were not created from a + :c:type:`PyModuleDef` structure. + .. c:macro:: PYTHON_API_VERSION + PYTHON_API_STRING - The C API version. Defined for backwards compatibility. + The C API version, as an integer (``1013``) and string (``"1013"``), respectively. + Defined for backwards compatibility. Currently, this constant is not updated in new Python versions, and is not useful for versioning. This may change in the future. .. c:macro:: PYTHON_ABI_VERSION + PYTHON_ABI_STRING - Defined as ``3`` for backwards compatibility. + Defined as ``3`` and ``"3"``, respectively, for backwards compatibility. Currently, this constant is not updated in new Python versions, and is not useful for versioning. This may change in the future. +.. _capi-module-support-functions: + Support functions ----------------- -The following functions are provided to help initialize a module -state. -They are intended for a module's execution slots (:c:data:`Py_mod_exec`), +The following functions are provided to help initialize a module object. +They are intended for a module's execution slot (:c:data:`Py_mod_exec`), the initialization function for legacy :ref:`single-phase initialization `, or code that creates modules dynamically. @@ -581,9 +1019,7 @@ or code that creates modules dynamically. // PyModule_AddObject() stole a reference to obj: // Py_XDECREF(obj) is not needed here. - .. deprecated:: 3.13 - - :c:func:`PyModule_AddObject` is :term:`soft deprecated`. + .. soft-deprecated:: 3.13 .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) @@ -645,6 +1081,9 @@ or code that creates modules dynamically. :c:type:`PyMethodDef` arrays; in that case they should call this function directly. + The *functions* array must be statically allocated (or otherwise guaranteed + to outlive the module object). + .. versionadded:: 3.5 .. c:function:: int PyModule_SetDocString(PyObject *module, const char *docstring) @@ -654,6 +1093,9 @@ or code that creates modules dynamically. ``PyModuleDef`` (such as when using :ref:`multi-phase-initialization`, ``PyModule_Create``, or ``PyModule_FromDefAndSpec``). + Return ``0`` on success. + Return ``-1`` with an exception set on error. + .. versionadded:: 3.5 .. c:function:: int PyUnstable_Module_SetGIL(PyObject *module, void *gil) diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index 7926148302af0b5..4bfcb86abf58ed1 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -136,7 +136,7 @@ Managing the Monitoring State ----------------------------- Monitoring states can be managed with the help of monitoring scopes. A scope -would typically correspond to a python function. +would typically correspond to a Python function. .. c:function:: int PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, const uint8_t *event_types, Py_ssize_t length) @@ -205,6 +205,4 @@ would typically correspond to a python function. .. versionadded:: 3.13 - .. deprecated:: 3.14 - - This function is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 55f0d0f9fb7ff86..eedeb180c6b7606 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -73,7 +73,7 @@ Object Protocol Flag to be used with multiple functions that print the object (like :c:func:`PyObject_Print` and :c:func:`PyFile_WriteObject`). - If passed, these function would use the :func:`str` of the object + If passed, these functions use the :func:`str` of the object instead of the :func:`repr`. @@ -85,6 +85,35 @@ Object Protocol instead of the :func:`repr`. +.. c:function:: void PyObject_Dump(PyObject *op) + + Dump an object *op* to ``stderr``. This should only be used for debugging. + + The output is intended to try dumping objects even after memory corruption: + + * Information is written starting with fields that are the least likely to + crash when accessed. + * This function can be called without an :term:`attached thread state`, but + it's not recommended to do so: it can cause deadlocks. + * An object that does not belong to the current interpreter may be dumped, + but this may also cause crashes or unintended behavior. + * Implement a heuristic to detect if the object memory has been freed. Don't + display the object contents in this case, only its memory address. + * The output format may change at any time. + + Example of output: + + .. code-block:: output + + object address : 0x7f80124702c0 + object refcount : 2 + object type : 0x9902e0 + object type name: str + object repr : 'abcdef' + + .. versionadded:: 3.15 + + .. c:function:: int PyObject_HasAttrWithError(PyObject *o, PyObject *attr_name) Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. @@ -197,11 +226,11 @@ Object Protocol in favour of using :c:func:`PyObject_DelAttr`, but there are currently no plans to remove it. - The function must not be called with ``NULL`` *v* and an an exception set. + The function must not be called with a ``NULL`` *v* and an exception set. This case can arise from forgetting ``NULL`` checks and would delete the attribute. - .. versionchanged:: next + .. versionchanged:: 3.15 Must not be called with NULL value if an exception is set. @@ -214,7 +243,7 @@ Object Protocol If *v* is ``NULL``, the attribute is deleted, but this feature is deprecated in favour of using :c:func:`PyObject_DelAttrString`. - The function must not be called with ``NULL`` *v* and an an exception set. + The function must not be called with a ``NULL`` *v* and an exception set. This case can arise from forgetting ``NULL`` checks and would delete the attribute. @@ -226,7 +255,7 @@ Object Protocol For more details, see :c:func:`PyUnicode_InternFromString`, which may be used internally to create a key object. - .. versionchanged:: next + .. versionchanged:: 3.15 Must not be called with NULL value if an exception is set. @@ -334,6 +363,8 @@ Object Protocol representation on success, ``NULL`` on failure. This is the equivalent of the Python expression ``repr(o)``. Called by the :func:`repr` built-in function. + If argument is ``NULL``, return the string ``''``. + .. versionchanged:: 3.4 This function now includes a debug assertion to help ensure that it does not silently discard an active exception. @@ -348,6 +379,8 @@ Object Protocol a string similar to that returned by :c:func:`PyObject_Repr` in Python 2. Called by the :func:`ascii` built-in function. + If argument is ``NULL``, return the string ``''``. + .. index:: string; PyObject_Str (C function) @@ -358,6 +391,8 @@ Object Protocol Python expression ``str(o)``. Called by the :func:`str` built-in function and, therefore, by the :func:`print` function. + If argument is ``NULL``, return the string ``''``. + .. versionchanged:: 3.4 This function now includes a debug assertion to help ensure that it does not silently discard an active exception. @@ -373,6 +408,8 @@ Object Protocol a TypeError is raised when *o* is an integer instead of a zero-initialized bytes object. + If argument is ``NULL``, return the :class:`bytes` object ``b''``. + .. c:function:: int PyObject_IsSubclass(PyObject *derived, PyObject *cls) @@ -600,7 +637,7 @@ Object Protocol Clear the managed dictionary of *obj*. - This function must only be called in a traverse function of the type which + This function must only be called in a clear function of the type which has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set. .. versionadded:: 3.13 @@ -682,10 +719,10 @@ Object Protocol :c:func:`PyUnstable_EnableTryIncRef` must have been called earlier on *obj* or this function may spuriously return ``0`` in the - :term:`free threading` build. + :term:`free-threaded build`. This function is logically equivalent to the following C code, except that - it behaves atomically in the :term:`free threading` build:: + it behaves atomically in the :term:`free-threaded build`:: if (Py_REFCNT(op) > 0) { Py_INCREF(op); @@ -762,13 +799,30 @@ Object Protocol On GIL-enabled builds, this function is equivalent to :c:expr:`Py_REFCNT(op) == 1`. - On a :term:`free threaded ` build, this checks if *op*'s + On a :term:`free-threaded build`, this checks if *op*'s :term:`reference count` is equal to one and additionally checks if *op* is only used by this thread. :c:expr:`Py_REFCNT(op) == 1` is **not** - thread-safe on free threaded builds; prefer this function. + thread-safe on free-threaded builds; prefer this function. The caller must hold an :term:`attached thread state`, despite the fact that this function doesn't call into the Python interpreter. This function cannot fail. .. versionadded:: 3.14 + +.. c:function:: int PyUnstable_SetImmortal(PyObject *op) + + Marks the object *op* :term:`immortal`. The argument should be uniquely referenced by + the calling thread. This is intended to be used for reducing reference counting contention + in the :term:`free-threaded build` for objects which are shared across threads. + + This is a one-way process: objects can only be made immortal; they cannot be + made mortal once again. Immortal objects do not participate in reference counting + and will never be garbage collected. If the object is GC-tracked, it is untracked. + + This function is intended to be used soon after *op* is created, by the code that + creates it, such as in the object's :c:member:`~PyTypeObject.tp_new` slot. + Returns 1 if the object was made immortal and returns 0 if it was not. + This function cannot fail. + + .. versionadded:: 3.15 diff --git a/Doc/c-api/perfmaps.rst b/Doc/c-api/perfmaps.rst index 76a1e9f528dc70c..a962c4ee09ad77d 100644 --- a/Doc/c-api/perfmaps.rst +++ b/Doc/c-api/perfmaps.rst @@ -31,7 +31,7 @@ Note that holding an :term:`attached thread state` is not required for these API or ``-2`` on failure to create a lock. Check ``errno`` for more information about the cause of a failure. -.. c:function:: int PyUnstable_WritePerfMapEntry(const void *code_addr, unsigned int code_size, const char *entry_name) +.. c:function:: int PyUnstable_WritePerfMapEntry(const void *code_addr, size_t code_size, const char *entry_name) Write one single entry to the ``/tmp/perf-$pid.map`` file. This function is thread safe. Here is what an example entry looks like:: @@ -49,3 +49,43 @@ Note that holding an :term:`attached thread state` is not required for these API This is called by the runtime itself during interpreter shut-down. In general, there shouldn't be a reason to explicitly call this, except to handle specific scenarios such as forking. + +.. c:function:: int PyUnstable_CopyPerfMapFile(const char *parent_filename) + + Open the ``/tmp/perf-$pid.map`` file and append the content of *parent_filename* + to it. + + This function is available on all platforms but only generates output on platforms + that support perf maps (currently only Linux). On other platforms, it does nothing. + + .. versionadded:: 3.13 + +.. c:function:: int PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *code) + + Compile the given code object using the current perf trampoline. + + The "current" trampoline is the one set by the runtime or the most recent + :c:func:`PyUnstable_PerfTrampoline_SetPersistAfterFork` call. + + If no trampoline is set, falls back to normal compilation (no perf map entry). + + :param code: The code object to compile. + :return: 0 on success, -1 on failure. + + .. versionadded:: 3.13 + +.. c:function:: int PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable) + + Set whether the perf trampoline should persist after a fork. + + * If ``enable`` is true (non-zero): perf map file remains open/valid post-fork. + Child process inherits all existing perf map entries. + * If ``enable`` is false (zero): perf map closes post-fork. + Child process gets empty perf map. + + Default: false (clears on fork). + + :param enable: 1 to enable, 0 to disable. + :return: 0 on success, -1 on failure. + + .. versionadded:: 3.13 diff --git a/Doc/c-api/picklebuffer.rst b/Doc/c-api/picklebuffer.rst new file mode 100644 index 000000000000000..9e2d92341b0f93e --- /dev/null +++ b/Doc/c-api/picklebuffer.rst @@ -0,0 +1,59 @@ +.. highlight:: c + +.. _picklebuffer-objects: + +.. index:: + pair: object; PickleBuffer + +Pickle buffer objects +--------------------- + +.. versionadded:: 3.8 + +A :class:`pickle.PickleBuffer` object wraps a :ref:`buffer-providing object +` for out-of-band data transfer with the :mod:`pickle` module. + + +.. c:var:: PyTypeObject PyPickleBuffer_Type + + This instance of :c:type:`PyTypeObject` represents the Python pickle buffer type. + This is the same object as :class:`pickle.PickleBuffer` in the Python layer. + + +.. c:function:: int PyPickleBuffer_Check(PyObject *op) + + Return true if *op* is a pickle buffer instance. + This function always succeeds. + + +.. c:function:: PyObject *PyPickleBuffer_FromObject(PyObject *obj) + + Create a pickle buffer from the object *obj*. + + This function will fail if *obj* doesn't support the :ref:`buffer protocol `. + + On success, return a new pickle buffer instance. + On failure, set an exception and return ``NULL``. + + Analogous to calling :class:`pickle.PickleBuffer` with *obj* in Python. + + +.. c:function:: const Py_buffer *PyPickleBuffer_GetBuffer(PyObject *picklebuf) + + Get a pointer to the underlying :c:type:`Py_buffer` that the pickle buffer wraps. + + The returned pointer is valid as long as *picklebuf* is alive and has not been + released. The caller must not modify or free the returned :c:type:`Py_buffer`. + If the pickle buffer has been released, raise :exc:`ValueError`. + + On success, return a pointer to the buffer view. + On failure, set an exception and return ``NULL``. + + +.. c:function:: int PyPickleBuffer_Release(PyObject *picklebuf) + + Release the underlying buffer held by the pickle buffer. + + Return ``0`` on success. On failure, set an exception and return ``-1``. + + Analogous to calling :meth:`pickle.PickleBuffer.release` in Python. diff --git a/Doc/c-api/profiling.rst b/Doc/c-api/profiling.rst new file mode 100644 index 000000000000000..0200f2eac6d9086 --- /dev/null +++ b/Doc/c-api/profiling.rst @@ -0,0 +1,239 @@ +.. highlight:: c + +.. _profiling: + +Profiling and tracing +===================== + +The Python interpreter provides some low-level support for attaching profiling +and execution tracing facilities. These are used for profiling, debugging, and +coverage analysis tools. + +This C interface allows the profiling or tracing code to avoid the overhead of +calling through Python-level callable objects, making a direct C function call +instead. The essential attributes of the facility have not changed; the +interface allows trace functions to be installed per-thread, and the basic +events reported to the trace function are the same as had been reported to the +Python-level trace functions in previous versions. + + +.. c:type:: int (*Py_tracefunc)(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) + + The type of the trace function registered using :c:func:`PyEval_SetProfile` and + :c:func:`PyEval_SetTrace`. The first parameter is the object passed to the + registration function as *obj*, *frame* is the frame object to which the event + pertains, *what* is one of the constants :c:data:`PyTrace_CALL`, + :c:data:`PyTrace_EXCEPTION`, :c:data:`PyTrace_LINE`, :c:data:`PyTrace_RETURN`, + :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION`, :c:data:`PyTrace_C_RETURN`, + or :c:data:`PyTrace_OPCODE`, and *arg* depends on the value of *what*: + + +-------------------------------+----------------------------------------+ + | Value of *what* | Meaning of *arg* | + +===============================+========================================+ + | :c:data:`PyTrace_CALL` | Always :c:data:`Py_None`. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_EXCEPTION` | Exception information as returned by | + | | :func:`sys.exc_info`. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_LINE` | Always :c:data:`Py_None`. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_RETURN` | Value being returned to the caller, | + | | or ``NULL`` if caused by an exception. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_C_CALL` | Function object being called. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_C_EXCEPTION` | Function object being called. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_C_RETURN` | Function object being called. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_OPCODE` | Always :c:data:`Py_None`. | + +-------------------------------+----------------------------------------+ + +.. c:var:: int PyTrace_CALL + + The value of the *what* parameter to a :c:type:`Py_tracefunc` function when a new + call to a function or method is being reported, or a new entry into a generator. + Note that the creation of the iterator for a generator function is not reported + as there is no control transfer to the Python bytecode in the corresponding + frame. + + +.. c:var:: int PyTrace_EXCEPTION + + The value of the *what* parameter to a :c:type:`Py_tracefunc` function when an + exception has been raised. The callback function is called with this value for + *what* when after any bytecode is processed after which the exception becomes + set within the frame being executed. The effect of this is that as exception + propagation causes the Python stack to unwind, the callback is called upon + return to each frame as the exception propagates. Only trace functions receive + these events; they are not needed by the profiler. + + +.. c:var:: int PyTrace_LINE + + The value passed as the *what* parameter to a :c:type:`Py_tracefunc` function + (but not a profiling function) when a line-number event is being reported. + It may be disabled for a frame by setting :attr:`~frame.f_trace_lines` to + *0* on that frame. + + +.. c:var:: int PyTrace_RETURN + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a + call is about to return. + + +.. c:var:: int PyTrace_C_CALL + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C + function is about to be called. + + +.. c:var:: int PyTrace_C_EXCEPTION + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C + function has raised an exception. + + +.. c:var:: int PyTrace_C_RETURN + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C + function has returned. + + +.. c:var:: int PyTrace_OPCODE + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions (but not + profiling functions) when a new opcode is about to be executed. This event is + not emitted by default: it must be explicitly requested by setting + :attr:`~frame.f_trace_opcodes` to *1* on the frame. + + +.. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) + + Set the profiler function to *func*. The *obj* parameter is passed to the + function as its first parameter, and may be any Python object, or ``NULL``. If + the profile function needs to maintain state, using a different value for *obj* + for each thread provides a convenient and thread-safe place to store it. The + profile function is called for all monitored events except :c:data:`PyTrace_LINE` + :c:data:`PyTrace_OPCODE` and :c:data:`PyTrace_EXCEPTION`. + + See also the :func:`sys.setprofile` function. + + The caller must have an :term:`attached thread state`. + + +.. c:function:: void PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *obj) + + Like :c:func:`PyEval_SetProfile` but sets the profile function in all running threads + belonging to the current interpreter instead of the setting it only on the current thread. + + The caller must have an :term:`attached thread state`. + + As :c:func:`PyEval_SetProfile`, this function ignores any exceptions raised while + setting the profile functions in all threads. + +.. versionadded:: 3.12 + + +.. c:function:: void PyEval_SetTrace(Py_tracefunc func, PyObject *obj) + + Set the tracing function to *func*. This is similar to + :c:func:`PyEval_SetProfile`, except the tracing function does receive line-number + events and per-opcode events, but does not receive any event related to C function + objects being called. Any trace function registered using :c:func:`PyEval_SetTrace` + will not receive :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION` or + :c:data:`PyTrace_C_RETURN` as a value for the *what* parameter. + + See also the :func:`sys.settrace` function. + + The caller must have an :term:`attached thread state`. + + +.. c:function:: void PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *obj) + + Like :c:func:`PyEval_SetTrace` but sets the tracing function in all running threads + belonging to the current interpreter instead of the setting it only on the current thread. + + The caller must have an :term:`attached thread state`. + + As :c:func:`PyEval_SetTrace`, this function ignores any exceptions raised while + setting the trace functions in all threads. + +.. versionadded:: 3.12 + + +Reference tracing +================= + +.. versionadded:: 3.13 + + +.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) + + The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`. + The first parameter is a Python object that has been just created (when **event** + is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event** + is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer + that was provided when :c:func:`PyRefTracer_SetTracer` was called. + + If a new tracing function is registered replacing the current one, a call to the + trace function will be made with the object set to **NULL** and **event** set to + :c:data:`PyRefTracer_TRACKER_REMOVED`. This will happen just before the new + function is registered. + +.. versionadded:: 3.13 + + +.. c:var:: int PyRefTracer_CREATE + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been created. + + +.. c:var:: int PyRefTracer_DESTROY + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been destroyed. + + +.. c:var:: int PyRefTracer_TRACKER_REMOVED + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when the + current tracer is about to be replaced by a new one. + + .. versionadded:: 3.14 + + +.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) + + Register a reference tracer function. The function will be called when a new + Python object has been created or when an object is going to be destroyed. If + **data** is provided it must be an opaque pointer that will be provided when + the tracer function is called. Return ``0`` on success. Set an exception and + return ``-1`` on error. + + Note that tracer functions **must not** create Python objects inside or + otherwise the call will be re-entrant. The tracer also **must not** clear + any existing exception or set an exception. A :term:`thread state` will be active + every time the tracer function is called. + + There must be an :term:`attached thread state` when calling this function. + + If another tracer function was already registered, the old function will be + called with **event** set to :c:data:`PyRefTracer_TRACKER_REMOVED` just before + the new function is registered. + +.. versionadded:: 3.13 + + +.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) + + Get the registered reference tracer function and the value of the opaque data + pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called. + If no tracer was registered this function will return NULL and will set the + **data** pointer to NULL. + + There must be an :term:`attached thread state` when calling this function. + +.. versionadded:: 3.13 diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 57a0728d4e9af42..4d56a92bf2af797 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -25,7 +25,7 @@ of Python objects. .. note:: - On :term:`free threaded ` builds of Python, returning 1 + On :term:`free-threaded builds ` of Python, returning 1 isn't sufficient to determine if it's safe to treat *o* as having no access by other threads. Use :c:func:`PyUnstable_Object_IsUniquelyReferenced` for that instead. diff --git a/Doc/c-api/sentinel.rst b/Doc/c-api/sentinel.rst new file mode 100644 index 000000000000000..b1b7329a5d42c59 --- /dev/null +++ b/Doc/c-api/sentinel.rst @@ -0,0 +1,47 @@ +.. highlight:: c + +.. _sentinelobjects: + +Sentinel objects +---------------- + +.. c:var:: PyTypeObject PySentinel_Type + + This instance of :c:type:`PyTypeObject` represents the Python + :class:`sentinel` type. This is the same object as :class:`sentinel`. + + .. versionadded:: 3.15 + +.. c:function:: int PySentinel_Check(PyObject *o) + + Return true if *o* is a :class:`sentinel` object or a subtype. + The :class:`sentinel` type does not currently allow subclasses, + so this check is exact. + Future Python versions may choose to allow subtyping. + This function always succeeds. + + .. versionadded:: 3.15 + +.. c:function:: int PySentinel_CheckExact(PyObject *o) + + Return true if *o* is a :class:`sentinel` object, but not a subtype. + The :class:`sentinel` type does not currently allow subclasses. + Future Python versions may choose to allow subtyping. + This function always succeeds. + + .. versionadded:: 3.15 + +.. c:function:: PyObject* PySentinel_New(const char *name, const char *module_name, const char *repr) + + Return a new :class:`sentinel` object with :attr:`~sentinel.__name__` set to + *name* and :attr:`~sentinel.__module__` set to *module_name*. + *name* must not be ``NULL``. If *module_name* is ``NULL``, :attr:`~sentinel.__module__` + is set to ``None``. If *repr* is ``NULL``, ``repr()`` returns :attr:`~sentinel.__name__`. + Return ``NULL`` with an exception set on failure. + + For pickling to work, *module_name* must be the name of an importable + module, and the sentinel must be accessible from that module under a + path matching *name*. Pickle treats *name* as a global variable name + in *module_name* (see :meth:`object.__reduce__`). + + .. versionadded:: 3.15 diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index df5bf6b64a93a00..6bae8f25ad75d15 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -109,9 +109,8 @@ Sequence Protocol Alias for :c:func:`PySequence_Contains`. - .. deprecated:: 3.14 - The function is :term:`soft deprecated` and should no longer be used to - write new code. + .. soft-deprecated:: 3.14 + The function should no longer be used to write new code. .. c:function:: Py_ssize_t PySequence_Index(PyObject *o, PyObject *value) diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index cba823aa027bd6d..db537aff2e6ce5a 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -5,9 +5,6 @@ Set Objects ----------- -.. sectionauthor:: Raymond D. Hettinger - - .. index:: pair: object; set pair: object; frozenset @@ -92,6 +89,11 @@ the constructor functions work with any iterable Python object. actually iterable. The constructor is also useful for copying a set (``c=set(s)``). + .. note:: + + The operation is atomic on :term:`free threading ` + when *iterable* is a :class:`set`, :class:`frozenset`, :class:`dict` or :class:`frozendict`. + .. c:function:: PyObject* PyFrozenSet_New(PyObject *iterable) @@ -100,6 +102,11 @@ the constructor functions work with any iterable Python object. set on success or ``NULL`` on failure. Raise :exc:`TypeError` if *iterable* is not actually iterable. + .. note:: + + The operation is atomic on :term:`free threading ` + when *iterable* is a :class:`set`, :class:`frozenset`, :class:`dict` or :class:`frozendict`. + The following functions and macros are available for instances of :class:`set` or :class:`frozenset` or instances of their subtypes. @@ -127,6 +134,10 @@ or :class:`frozenset` or instances of their subtypes. the *key* is unhashable. Raise :exc:`SystemError` if *anyset* is not a :class:`set`, :class:`frozenset`, or an instance of a subtype. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. .. c:function:: int PySet_Add(PyObject *set, PyObject *key) @@ -138,6 +149,12 @@ or :class:`frozenset` or instances of their subtypes. :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + + The following functions are available for instances of :class:`set` or its subtypes but not for instances of :class:`frozenset` or its subtypes. @@ -147,11 +164,16 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. Return ``1`` if found and removed, ``0`` if not found (no action taken), and ``-1`` if an error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a - :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~frozenset.discard` + :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~set.discard` method, this function does not automatically convert unhashable sets into temporary frozensets. Raise :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. c:function:: PyObject* PySet_Pop(PyObject *set) @@ -166,3 +188,28 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. Empty an existing set of all elements. Return ``0`` on success. Return ``-1`` and raise :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. + + .. note:: + + In the :term:`free-threaded build`, the set is emptied before its entries + are cleared, so other threads will observe an empty set rather than + intermediate states. + + +Deprecated API +^^^^^^^^^^^^^^ + +.. c:macro:: PySet_MINSIZE + + A constant representing the size of an internal + preallocated table inside :c:type:`PySetObject` instances. + + This is documented solely for completeness, as there are no guarantees + that a given version of CPython uses preallocated tables with a fixed + size. + In code that does not deal with unstable set internals, + :c:macro:`!PySet_MINSIZE` can be replaced with a small constant like ``8``. + + If looking for the size of a set, use :c:func:`PySet_Size` instead. + + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/slots.rst b/Doc/c-api/slots.rst new file mode 100644 index 000000000000000..84a125cb60bae7e --- /dev/null +++ b/Doc/c-api/slots.rst @@ -0,0 +1,240 @@ +.. highlight:: c + +.. _capi-slots: + +Definition slots +================ + +To define :ref:`module objects ` and +:ref:`classes ` using the C API, you may use +an array of *slots* -- essentially, key-value pairs that describe features +of the object to create. +This decouples the data from the structures used at runtime, allowing CPython +-- and other Python C API implementations -- to update the structures without +breaking backwards compatibility. + +This section documents slots in general. +For object-specific behavior and slot values, see documentation for functions +that apply slots: + +- :c:func:`PyType_FromSlots` for types; +- :c:func:`PyModule_FromSlotsAndSpec` and :ref:`extension-export-hook` + for modules. + +When slots are passed to a function that applies them, the function will +not modify the slot array, nor any data it points to (recursively). +After the function is done, the caller is allowed to modify or deallocate +the array and any data it points to (recursively), except data +explicitly marked with :c:macro:`PySlot_STATIC`. + +Except when documented otherwise, multiple slots with the same ID +(:c:member:`~PySlot.sl_id`) may not occur in a single slots array. + +.. versionadded:: 3.15 + + Slot arrays generalize an earlier way of defining objects: + using :c:type:`PyType_Spec` with :c:type:`PyType_Slot` for types, and + :c:type:`PyModuleDef` with :c:type:`PyModuleDef_Slot` for modules. + The earlier API is :term:`soft deprecated`; there are no plans to remove it. + +Entries of the slots array use the following structure: + +.. c:type:: PySlot + + An entry in a slots array. Defined as: + + .. code-block:: c + + typedef struct { + uint16_t sl_id; + uint16_t sl_flags; + uint32_t _reserved; // must be 0 + union { + void *sl_ptr; + void (*sl_func)(void); + Py_ssize_t sl_size; + int64_t sl_int64; + uint64_t sl_uint64; + }; + } PySlot; + + .. c:member:: uint16_t sl_id + + A slot ID, chosen from: + + - ``Py_slot_*`` values documented in :ref:`pyslot-common-ids` below; + - ``Py_mod_*`` values for modules, as documented in :ref:`c_module_slots`; + - Values for types, as documented in :ref:`pyslot_type_slot_ids`. + + A :c:member:`!sl_id` of zero (:c:macro:`Py_slot_end`) marks the end of a + slots array. + + .. c:member:: void *sl_ptr + void (*sl_func)(void) + Py_ssize_t sl_size + int64_t sl_int64 + uint64_t sl_uint64 + + The data for the slot. + These members are part of an anonymous union; + the member to use depends on which data type is required by the slot ID: + data pointer, function pointer, size, signed or unsigned + integer, respectively. + + Except when documented otherwise for a specific slot ID, pointers + (that is :c:member:`!sl_ptr` and :c:member:`!sl_func`) may not be NULL. + + .. c:member:: uint16_t sl_flags + + Zero or more of the following flags, OR-ed together: + + .. c:namespace:: NULL + + .. c:macro:: PySlot_STATIC + + All data the slot points to is statically allocated and constant. + Thus, the interpreter does not need to copy the information. + + This flag is implied for function pointers. + + The flag applies even to data the slot points to “indirectly”, + except for slots nested via :c:macro:`Py_slot_subslots` which may + have their own :c:macro:`!PySlot_STATIC` flags. + For example, if applied to a :c:macro:`Py_tp_members` slot that + points to an array of :c:type:`PyMemberDef` structures, + then the entire array, as well as the name and doc strings + in its elements, must be static and constant. + + .. c:macro:: PySlot_INTPTR + + The data is stored in ``sl_ptr``; CPython will cast it to + the appropriate type. + + This flag can simplify porting from the older :c:type:`PyType_Slot` + and :c:type:`PyModuleDef_Slot` structures. + + .. c:macro:: PySlot_OPTIONAL + + If the slot ID is unknown, the interpreter should ignore the + slot, rather than fail. + + For example, if Python 3.16 adds a new feature with a new slot ID,attr + the corresponding slot may be marked :c:macro:`!PySlot_OPTIONAL` + so that Python 3.15 ignores it. + + Note that the "optionality" only applies to unknown slot IDs. + This flag does not make Python skip invalid values of known slots. + + .. versionadded:: 3.15 + + +Convenience macros +------------------ + +.. c:macro:: PySlot_DATA(name, value) + PySlot_FUNC(name, value) + PySlot_SIZE(name, value) + PySlot_INT64(name, value) + PySlot_UINT64(name, value) + PySlot_STATIC_DATA(name, value) + + Convenience macros to define :c:type:`!PySlot` structures with + :c:member:`~PySlot.sl_id` and a particular union member set. + + :c:macro:`!PySlot_STATIC_DATA` sets the :c:macro:`PySlot_STATIC` flag; + others set no flags. + + Note that these macros use *designated initializers*, a C language feature + that C++ added in the 2020 version of the standard. + If your code needs to be compatible with C++11 or older, + use :c:macro:`PySlot_PTR` instead. + + Defined as:: + + #define PySlot_DATA(NAME, VALUE) \ + {.sl_id=NAME, .sl_ptr=(void*)(VALUE)} + + #define PySlot_FUNC(NAME, VALUE) \ + {.sl_id=NAME, .sl_func=(VALUE)} + + #define PySlot_SIZE(NAME, VALUE) \ + {.sl_id=NAME, .sl_size=(VALUE)} + + #define PySlot_INT64(NAME, VALUE) \ + {.sl_id=NAME, .sl_int64=(VALUE)} + + #define PySlot_UINT64(NAME, VALUE) \ + {.sl_id=NAME, .sl_uint64=(VALUE)} + + #define PySlot_STATIC_DATA(NAME, VALUE) \ + {.sl_id=NAME, .sl_flags=PySlot_STATIC, .sl_ptr=(VALUE)} + + .. versionadded:: 3.15 + +.. c:macro:: PySlot_END + + Convenience macro to mark the end of a :c:type:`!PySlot` array. + + Defined as:: + + #define PySlot_END {0} + + .. versionadded:: 3.15 + +.. c:macro:: PySlot_PTR(name, value) + PySlot_PTR_STATIC(name, value) + + Convenience macros for use in C++11-compatible code. + This version of C++ does not allow setting arbitrary union members in + literals; instead, these macros set the :c:macro:`PySlot_INTPTR` flag and cast + the value to ``(void*)``. + + Defined as:: + + #define PySlot_PTR(NAME, VALUE) \ + {NAME, PySlot_INTPTR, {0}, {(void*)(VALUE)}} + + #define PySlot_PTR_STATIC(NAME, VALUE) \ + {NAME, PySlot_INTPTR|Py_SLOT_STATIC, {0}, {(void*)(VALUE)}} + + .. versionadded:: 3.15 + +.. _pyslot-common-ids: + +Common slot IDs +--------------- + +The following slot IDs may be used in both type and module definitions. + +.. c:macro:: Py_slot_end + + Marks the end of a slots array. + Defined as zero. + + .. versionadded:: 3.15 + +.. c:macro:: Py_slot_subslots + + Nested slots array. + + The value (:c:member:`~PySlot.sl_ptr`) should point to an array of + :c:type:`PySlot` structures. + The slots in the array (up to but not including the zero-ID + terminator) will be treated as if they were inserted if the current + slot array, at the point :c:macro:`!Py_slot_subslots` appears. + + Slot nesting depth is limited to 5 levels. + This restriction may be lifted in the future. + + .. versionadded:: 3.15 + +.. c:macro:: Py_slot_invalid + + Reserved; will always be treated as an unknown slot ID. + Defined as ``UINT16_MAX`` (``0xFFFF``). + + When used with the :c:macro:`PySlot_OPTIONAL` flag, defines a slot with + no effect. + Without the flag, processing a slot with this ID will fail. + + .. versionadded:: 3.15 diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 9b65e0b8d23d93f..13e5d5c96135c0e 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -2,9 +2,9 @@ .. _stable: -*************** -C API Stability -*************** +*********************** +C API and ABI Stability +*********************** Unless documented otherwise, Python's C API is covered by the Backwards Compatibility Policy, :pep:`387`. @@ -51,135 +51,212 @@ It is generally intended for specialized, low-level tools like debuggers. Projects that use this API are expected to follow CPython development and spend extra effort adjusting to changes. +.. _stable-abi: .. _stable-application-binary-interface: -Stable Application Binary Interface -=================================== +Stable Application Binary Interfaces +==================================== -For simplicity, this document talks about *extensions*, but the Limited API -and Stable ABI work the same way for all uses of the API – for example, -embedding Python. +Python's :dfn:`Stable ABI` allows extensions to be compatible with multiple +versions of Python, without recompilation. -.. _limited-c-api: +.. note:: -Limited C API -------------- + For simplicity, this document talks about *extensions*, but Stable ABI + works the same way for all uses of the API – for example, embedding Python. -Python 3.2 introduced the *Limited API*, a subset of Python's C API. -Extensions that only use the Limited API can be -compiled once and be loaded on multiple versions of Python. -Contents of the Limited API are :ref:`listed below `. +There are two Stable ABIs: -.. c:macro:: Py_LIMITED_API +- ``abi3``, introduced in Python 3.2, is compatible with + **non**-:term:`free-threaded ` builds of CPython. - Define this macro before including ``Python.h`` to opt in to only use - the Limited API, and to select the Limited API version. +- ``abi3t``, introduced in Python 3.15, is compatible with + :term:`free-threaded ` builds of CPython. + It has stricter API limitations than ``abi3``. - Define ``Py_LIMITED_API`` to the value of :c:macro:`PY_VERSION_HEX` - corresponding to the lowest Python version your extension supports. - The extension will be ABI-compatible with all Python 3 releases - from the specified one onward, and can use Limited API introduced up to that - version. + .. versionadded:: 3.15 - Rather than using the ``PY_VERSION_HEX`` macro directly, hardcode a minimum - minor version (e.g. ``0x030A0000`` for Python 3.10) for stability when - compiling with future Python versions. + ``abi3t`` was added in :pep:`803` - You can also define ``Py_LIMITED_API`` to ``3``. This works the same as - ``0x03020000`` (Python 3.2, the version that introduced Limited API). +It is possible for an extension to be compiled for *both* ``abi3`` and +``abi3t`` at the same time; the result will be compatible with +both free-threaded and non-free-threaded builds of Python. +Currently, this has no downsides compared to compiling for ``abi3t`` only. +Each Stable ABI is versioned using the first two numbers of the Python version. +For example, Stable ABI 3.14 corresponds to Python 3.14. +An extension compiled for Stable ABI 3.x is ABI-compatible with Python 3.x +and above. + +Extensions that target a stable ABI must only use a limited subset of +the C API. This subset is known as the :dfn:`Limited API`; its contents +are :ref:`listed below `. + +On Windows, extensions that use a Stable ABI should be linked against +``python3.dll`` rather than a version-specific library such as +``python39.dll``. +This library only exposes the relevant symbols. + +On some platforms, Python will look for and load shared library files named +with the ``abi3`` or ``abi3t`` tag (for example, ``mymodule.abi3.so``). +:term:`Free-threaded ` interpreters only recognize the +``abi3t`` tag, while non-free-threaded ones will prefer ``abi3`` but fall back +to ``abi3t``. +Thus, extensions compatible with both ABIs should use the ``abi3t`` tag. + +Python does not necessarily check that extensions it loads +have compatible ABI. +Extension authors are encouraged to add a check using the :c:macro:`Py_mod_abi` +slot or the :c:func:`PyABIInfo_Check` function, but the user +(or their packaging tool) is ultimately responsible for ensuring that, +for example, extensions built for Stable ABI 3.10 are not installed for lower +versions of Python. + +All functions in Stable ABI are present as functions in Python's shared +library, not solely as macros. +This makes them usable in languages that don't use the C +preprocessor, including Python's :py:mod:`ctypes`. -.. _stable-abi: -Stable ABI ----------- +.. _abi3-compiling: -To enable this, Python provides a *Stable ABI*: a set of symbols that will -remain ABI-compatible across Python 3.x versions. +Compiling for Stable ABI +------------------------ .. note:: - The Stable ABI prevents ABI issues, like linker errors due to missing - symbols or data corruption due to changes in structure layouts or function - signatures. - However, other changes in Python can change the *behavior* of extensions. - See Python's Backwards Compatibility Policy (:pep:`387`) for details. + Build tools (such as, for example, meson-python, scikit-build-core, + or Setuptools) often have a mechanism for setting macros and synchronizing + them with extension filenames and other metadata. + Prefer using such a mechanism, if it exists, over defining the + macros manually. -The Stable ABI contains symbols exposed in the :ref:`Limited API -`, but also other ones – for example, functions necessary to -support older versions of the Limited API. + The rest of this section is mainly relevant for tool authors, and for + people who compile extensions manually. -On Windows, extensions that use the Stable ABI should be linked against -``python3.dll`` rather than a version-specific library such as -``python39.dll``. + .. seealso:: `list of recommended tools`_ in the Python Packaging User Guide -On some platforms, Python will look for and load shared library files named -with the ``abi3`` tag (e.g. ``mymodule.abi3.so``). -It does not check if such extensions conform to a Stable ABI. -The user (or their packaging tools) need to ensure that, for example, -extensions built with the 3.10+ Limited API are not installed for lower -versions of Python. + .. _list of recommended tools: https://packaging.python.org/en/latest/guides/tool-recommendations/#build-backends-for-extension-modules + +To compile for a Stable ABI, define one or both of the following macros +to the lowest Python version your extension should support, in +:c:macro:`Py_PACK_VERSION` format. +Typically, you should choose a specific value rather than the version of +the Python headers you are compiling against. + +The macros must be defined before including ``Python.h``. +Since :c:macro:`Py_PACK_VERSION` is not available at this point, you +will need to use the numeric value directly. +For reference, the values for a few recent Python versions are: + +.. version-hex-cheatsheet:: + +When one of the macros is defined, ``Python.h`` will only expose API that is +compatible with the given Stable ABI -- that is, the +:ref:`Limited API ` plus some definitions that need to be +visible to the compiler but should not be used directly. +When both are defined, ``Python.h`` will only expose API compatible with +both Stable ABIs. + +.. c:macro:: Py_LIMITED_API + + Target ``abi3``, that is, + non-:term:`free-threaded builds ` of CPython. + See :ref:`above ` for common information. + +.. c:macro:: Py_TARGET_ABI3T -All functions in the Stable ABI are present as functions in Python's shared -library, not solely as macros. This makes them usable from languages that don't -use the C preprocessor. + Target ``abi3t``, that is, + :term:`free-threaded builds ` of CPython. + See :ref:`above ` for common information. + .. versionadded:: 3.15 -Limited API Scope and Performance ---------------------------------- +Both macros specify a target ABI; the different naming style is due to +backwards compatibility. -The goal for the Limited API is to allow everything that is possible with the +.. admonition:: Historical note + + You can also define ``Py_LIMITED_API`` as ``3``. This works the same as + ``0x03020000`` (Python 3.2, the version that introduced Stable ABI). + +When both are defined, ``Python.h`` may, or may not, redefine +:c:macro:`!Py_LIMITED_API` to match :c:macro:`!Py_TARGET_ABI3T`. + +On a :term:`free-threaded build` -- that is, when +:c:macro:`Py_GIL_DISABLED` is defined -- :c:macro:`!Py_TARGET_ABI3T` +defaults to the value of :c:macro:`!Py_LIMITED_API`. +This means that there are two ways to build for both ``abi3`` and ``abi3t``: + +- define both :c:macro:`!Py_LIMITED_API` and :c:macro:`!Py_TARGET_ABI3T`, or +- define only :c:macro:`!Py_LIMITED_API` and: + + - on Windows, define :c:macro:`!Py_GIL_DISABLED`; + - on other systems, use the headers of free-threaded build of Python. + + +.. _limited-api-scope-and-performance: + +Stable ABI Scope and Performance +-------------------------------- + +The goal for Stable ABI is to allow everything that is possible with the full C API, but possibly with a performance penalty. +Generally, compatibility with Stable ABI will require some changes to an +extension's source code. -For example, while :c:func:`PyList_GetItem` is available, its “unsafe” macro +For example, while :c:func:`PyList_GetItem` is available, its "unsafe" macro variant :c:func:`PyList_GET_ITEM` is not. The macro can be faster because it can rely on version-specific implementation details of the list object. -Without ``Py_LIMITED_API`` defined, some C API functions are inlined or -replaced by macros. -Defining ``Py_LIMITED_API`` disables this inlining, allowing stability as +For another example, when *not* compiling for Stable ABI, some C API +functions are inlined or replaced by macros. +Compiling for Stable ABI disables this inlining, allowing stability as Python's data structures are improved, but possibly reducing performance. -By leaving out the ``Py_LIMITED_API`` definition, it is possible to compile -a Limited API extension with a version-specific ABI. This can improve -performance for that Python version, but will limit compatibility. -Compiling with ``Py_LIMITED_API`` will then yield an extension that can be -distributed where a version-specific one is not available – for example, -for prereleases of an upcoming Python version. +By leaving out the :c:macro:`!Py_LIMITED_API` or :c:macro:`!Py_TARGET_ABI3T` +definition, it is possible to compile Stable-ABI-compatible source +for a version-specific ABI. +A potentially faster version-specific extension can then be distributed +alongside a version compiled for Stable ABI -- a slower but more compatible +fallback. -Limited API Caveats -------------------- +.. _limited-api-caveats: -Note that compiling with ``Py_LIMITED_API`` is *not* a complete guarantee that -code conforms to the :ref:`Limited API ` or the :ref:`Stable ABI -`. ``Py_LIMITED_API`` only covers definitions, but an API also -includes other issues, such as expected semantics. +Stable ABI Caveats +------------------ -One issue that ``Py_LIMITED_API`` does not guard against is calling a function -with arguments that are invalid in a lower Python version. +Note that compiling for Stable ABI is *not* a complete guarantee that code will +be compatible with the expected Python versions. +Stable ABI prevents *ABI* issues, like linker errors due to missing +symbols or data corruption due to changes in structure layouts or function +signatures. +However, other changes in Python can change the *behavior* of extensions. + +One issue that the :c:macro:`Py_TARGET_ABI3T` and :c:macro:`Py_LIMITED_API` +macros do not guard against is calling a function with arguments that are +invalid in a lower Python version. For example, consider a function that starts accepting ``NULL`` for an argument. In Python 3.9, ``NULL`` now selects a default behavior, but in Python 3.8, the argument will be used directly, causing a ``NULL`` dereference and crash. A similar argument works for fields of structs. -Another issue is that some struct fields are currently not hidden when -``Py_LIMITED_API`` is defined, even though they're part of the Limited API. - For these reasons, we recommend testing an extension with *all* minor Python -versions it supports, and preferably to build with the *lowest* such version. +versions it supports. We also recommend reviewing documentation of all used API to check if it is explicitly part of the Limited API. Even with ``Py_LIMITED_API`` defined, a few private declarations are exposed for technical reasons (or even unintentionally, as bugs). -Also note that the Limited API is not necessarily stable: compiling with -``Py_LIMITED_API`` with Python 3.8 means that the extension will -run with Python 3.12, but it will not necessarily *compile* with Python 3.12. -In particular, parts of the Limited API may be deprecated and removed, -provided that the Stable ABI stays stable. +Also note that while compiling with ``Py_LIMITED_API`` 3.8 means that the +extension should *load* on Python 3.12, and *compile* with Python 3.12, +the same source will not necessarily compile with ``Py_LIMITED_API`` +set to 3.12. +In general, parts of the Limited API may be deprecated and removed, +provided that Stable ABI stays stable. .. _stable-abi-platform: @@ -189,22 +266,187 @@ Platform Considerations ABI stability depends not only on Python, but also on the compiler used, lower-level libraries and compiler options. For the purposes of -the :ref:`Stable ABI `, these details define a “platform”. They +the :ref:`Stable ABIs `, these details define a “platform”. They usually depend on the OS type and processor architecture It is the responsibility of each particular distributor of Python to ensure that all Python versions on a particular platform are built -in a way that does not break the Stable ABI. +in a way that does not break the Stable ABIs, or the version-specific ABIs. This is the case with Windows and macOS releases from ``python.org`` and many third-party distributors. +ABI Checking +============ + +.. versionadded:: 3.15 + +Python includes a rudimentary check for ABI compatibility. + +This check is not comprehensive. +It only guards against common cases of incompatible modules being +installed for the wrong interpreter. +It also does not take :ref:`platform incompatibilities ` +into account. +It can only be done after an extension is successfully loaded. + +Despite these limitations, it is recommended that extension modules use this +mechanism, so that detectable incompatibilities raise exceptions rather than +crash. + +Most modules can use this check via the :c:data:`Py_mod_abi` +slot and the :c:macro:`PyABIInfo_VAR` macro, for example like this: + +.. code-block:: c + + PyABIInfo_VAR(abi_info); + + static PyModuleDef_Slot mymodule_slots[] = { + {Py_mod_abi, &abi_info}, + ... + }; + + +The full API is described below for advanced use cases. + +.. c:function:: int PyABIInfo_Check(PyABIInfo *info, const char *module_name) + + Verify that the given *info* is compatible with the currently running + interpreter. + + Return 0 on success. On failure, raise an exception and return -1. + + If the ABI is incompatible, the raised exception will be :py:exc:`ImportError`. + + The *module_name* argument can be ``NULL``, or point to a NUL-terminated + UTF-8-encoded string used for error messages. + + Note that if *info* describes the ABI that the current code uses (as defined + by :c:macro:`PyABIInfo_VAR`, for example), using any other Python C API + may lead to crashes. + In particular, it is not safe to examine the raised exception. + + .. versionadded:: 3.15 + +.. c:macro:: PyABIInfo_VAR(NAME) + + Define a static :c:struct:`PyABIInfo` variable with the given *NAME* that + describes the ABI that the current code will use. + This macro expands to: + + .. code-block:: c + + static PyABIInfo NAME = { + 1, 0, + PyABIInfo_DEFAULT_FLAGS, + PY_VERSION_HEX, + PyABIInfo_DEFAULT_ABI_VERSION + } + + .. versionadded:: 3.15 + +.. c:type:: PyABIInfo + + .. c:member:: uint8_t abiinfo_major_version + + The major version of :c:struct:`PyABIInfo`. Can be set to: + + * ``0`` to skip all checking, or + * ``1`` to specify this version of :c:struct:`!PyABIInfo`. + + .. c:member:: uint8_t abiinfo_minor_version + + The minor version of :c:struct:`PyABIInfo`. + Must be set to ``0``; larger values are reserved for backwards-compatible + future versions of :c:struct:`!PyABIInfo`. + + .. c:member:: uint16_t flags + + .. c:namespace:: NULL + + This field is usually set to the following macro: + + .. c:macro:: PyABIInfo_DEFAULT_FLAGS + + Default flags, based on current values of macros such as + :c:macro:`Py_LIMITED_API` and :c:macro:`Py_GIL_DISABLED`. + + Alternately, the field can be set to the following flags, combined + by bitwise OR. + Unused bits must be set to zero. + + ABI variant -- one of: + + .. c:macro:: PyABIInfo_STABLE + + Specifies that Stable ABI is used. + + .. c:macro:: PyABIInfo_INTERNAL + + Specifies ABI specific to a particular build of CPython. + Internal use only. + + Free-threading compatibility -- one of: + + .. c:macro:: PyABIInfo_FREETHREADED + + Specifies ABI compatible with :term:`free-threaded builds + ` of CPython. + (That is, ones compiled with :option:`--disable-gil`; with ``t`` + in :py:data:`sys.abiflags`) + + .. c:macro:: PyABIInfo_GIL + + Specifies ABI compatible with non-free-threaded builds of CPython + (ones compiled *without* :option:`--disable-gil`). + + .. c:macro:: PyABIInfo_FREETHREADING_AGNOSTIC + + Specifies ABI compatible with both free-threaded and + non-free-threaded builds of CPython, that is, both + ``abi3`` and ``abi3t``. + + .. c:member:: uint32_t build_version + + The version of the Python headers used to build the code, in the format + used by :c:macro:`PY_VERSION_HEX`. + + This can be set to ``0`` to skip any checks related to this field. + This option is meant mainly for projects that do not use the CPython + headers directly, and do not emulate a specific version of them. + + .. c:member:: uint32_t abi_version + + The ABI version. + + For Stable ABI, this field should be the value of + :c:macro:`Py_LIMITED_API` or :c:macro:`Py_TARGET_ABI3T`. + If both are defined, use the smaller value. + (If :c:macro:`Py_LIMITED_API` is ``3``; use + :c:expr:`Py_PACK_VERSION(3, 2)` instead of ``3``.) + + Otherwise, it should be set to :c:macro:`PY_VERSION_HEX`. + + It can also be set to ``0`` to skip any checks related to this field. + + .. c:namespace:: NULL + + .. c:macro:: PyABIInfo_DEFAULT_ABI_VERSION + + The value that should be used for this field, based on current + values of macros such as :c:macro:`Py_LIMITED_API`, + :c:macro:`PY_VERSION_HEX` and :c:macro:`Py_GIL_DISABLED`. + + .. versionadded:: 3.15 + + +.. _limited-c-api: .. _limited-api-list: Contents of Limited API ======================= - -Currently, the :ref:`Limited API ` includes the following items: +This is the definitive list of :ref:`Limited API ` for +Python |version|: .. limited-api-list:: diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 58dd915e04f619a..1ddde5d136b3bb1 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -33,6 +33,13 @@ under :ref:`reference counting `. The members must not be accessed directly; instead use macros such as :c:macro:`Py_REFCNT` and :c:macro:`Py_TYPE`. + In the :ref:`Stable ABI ` for Free-Threaded Builds (``abi3t``), + this struct is opaque; its size and layout may change between + Python versions. + In Stable ABI for non-free-threaded builds (``abi3``), the + :c:member:`!ob_refcnt` and :c:member:`!ob_type` fields are available, + but using them directly is discouraged. + .. c:member:: Py_ssize_t ob_refcnt The object's reference count, as returned by :c:macro:`Py_REFCNT`. @@ -48,6 +55,19 @@ under :ref:`reference counting `. Do not use this field directly; use :c:macro:`Py_TYPE` and :c:func:`Py_SET_TYPE` instead. + .. c:member:: PyMutex ob_mutex + + A :ref:`per-object lock `, present only in the :term:`free-threaded ` + build (when :c:macro:`Py_GIL_DISABLED` is defined). + + This field is **reserved for use by the critical section API** + (:c:macro:`Py_BEGIN_CRITICAL_SECTION` / :c:macro:`Py_END_CRITICAL_SECTION`). + Do **not** lock it directly with ``PyMutex_Lock``; doing so can cause + deadlocks. If you need your own lock, add a separate :c:type:`PyMutex` + field to your object struct. + + .. versionadded:: 3.13 + .. c:type:: PyVarObject @@ -59,6 +79,19 @@ under :ref:`reference counting `. instead use macros such as :c:macro:`Py_SIZE`, :c:macro:`Py_REFCNT` and :c:macro:`Py_TYPE`. + In the :ref:`Stable ABI ` for Free-Threaded Builds (``abi3t``), + this struct is opaque; its size and layout may change between + Python versions. + In Stable ABI for non-free-threaded builds (``abi3``), the + :c:member:`!ob_base` and :c:member:`!ob_size` fields are available, + but using them directly is discouraged. + + .. c:member:: PyObject ob_base + + Common object header. + Typically, this field is not accessed directly; instead + :c:type:`!PyVarObject` can be cast to :c:type:`PyObject`. + .. c:member:: Py_ssize_t ob_size A size field, whose contents should be considered an object's internal @@ -280,6 +313,8 @@ Implementing functions and methods Name of the method. + A ``NULL`` *ml_name* marks the end of a :c:type:`!PyMethodDef` array. + .. c:member:: PyCFunction ml_meth Pointer to the C implementation. @@ -408,7 +443,7 @@ There are these calling conventions: These two constants are not used to indicate the calling convention but the -binding when use with methods of classes. These may not be used for functions +binding when used with methods of classes. These may not be used for functions defined for modules. At most one of these flags may be set for any given method. @@ -419,8 +454,8 @@ method. The method will be passed the type object as the first parameter rather than an instance of the type. This is used to create *class methods*, - similar to what is created when using the :func:`classmethod` built-in - function. + similar to what is created when using the :deco:`classmethod` built-in + decorator. .. c:macro:: METH_STATIC @@ -429,7 +464,7 @@ method. The method will be passed ``NULL`` as the first parameter rather than an instance of the type. This is used to create *static methods*, similar to - what is created when using the :func:`staticmethod` built-in function. + what is created when using the :deco:`staticmethod` built-in decorator. One other constant controls whether a method is loaded in place of another definition with the same method name. @@ -447,6 +482,25 @@ definition with the same method name. slot. This is helpful because calls to PyCFunctions are optimized more than wrapper object calls. + +.. c:var:: PyTypeObject PyCMethod_Type + + The type object corresponding to Python C method objects. This is + available as :class:`types.BuiltinMethodType` in the Python layer. + + +.. c:function:: int PyCMethod_Check(PyObject *op) + + Return true if *op* is an instance of the :c:type:`PyCMethod_Type` type + or a subtype of it. This function always succeeds. + + +.. c:function:: int PyCMethod_CheckExact(PyObject *op) + + This is the same as :c:func:`PyCMethod_Check`, but does not account for + subtypes. + + .. c:function:: PyObject * PyCMethod_New(PyMethodDef *ml, PyObject *self, PyObject *module, PyTypeObject *cls) Turn *ml* into a Python :term:`callable` object. @@ -472,6 +526,24 @@ definition with the same method name. .. versionadded:: 3.9 +.. c:var:: PyTypeObject PyCFunction_Type + + The type object corresponding to Python C function objects. This is + available as :class:`types.BuiltinFunctionType` in the Python layer. + + +.. c:function:: int PyCFunction_Check(PyObject *op) + + Return true if *op* is an instance of the :c:type:`PyCFunction_Type` type + or a subtype of it. This function always succeeds. + + +.. c:function:: int PyCFunction_CheckExact(PyObject *op) + + This is the same as :c:func:`PyCFunction_Check`, but does not account for + subtypes. + + .. c:function:: PyObject * PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module) Equivalent to ``PyCMethod_New(ml, self, module, NULL)``. @@ -482,6 +554,62 @@ definition with the same method name. Equivalent to ``PyCMethod_New(ml, self, NULL, NULL)``. +.. c:function:: int PyCFunction_GetFlags(PyObject *func) + + Get the function's flags on *func* as they were passed to + :c:member:`~PyMethodDef.ml_flags`. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns the function's flags on success, and ``-1`` with an + exception set on failure. + + +.. c:function:: int PyCFunction_GET_FLAGS(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetFlags`, but without error + or type checking. + + +.. c:function:: PyCFunction PyCFunction_GetFunction(PyObject *func) + + Get the function pointer on *func* as it was passed to + :c:member:`~PyMethodDef.ml_meth`. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns the function pointer on success, and ``NULL`` with an + exception set on failure. + + +.. c:function:: int PyCFunction_GET_FUNCTION(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetFunction`, but without error + or type checking. + + +.. c:function:: PyObject *PyCFunction_GetSelf(PyObject *func) + + Get the "self" object on *func*. This is the object that would be passed + to the first argument of a :c:type:`PyCFunction`. For C function objects + created through a :c:type:`PyMethodDef` on a :c:type:`PyModuleDef`, this + is the resulting module object. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns a :term:`borrowed reference` to the "self" object + on success, and ``NULL`` with an exception set on failure. + + +.. c:function:: PyObject *PyCFunction_GET_SELF(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetSelf`, but without error or + type checking. + + Accessing attributes of extension types --------------------------------------- @@ -605,14 +733,12 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: entry indicates an offset from the subclass-specific data, rather than from ``PyObject``. - Can only be used as part of :c:member:`Py_tp_members ` + Can only be used as part of the :c:data:`Py_tp_members` :c:type:`slot ` when creating a class using negative :c:member:`~PyType_Spec.basicsize`. It is mandatory in that case. - - This flag is only used in :c:type:`PyType_Slot`. - When setting :c:member:`~PyTypeObject.tp_members` during - class creation, Python clears it and sets + When setting :c:member:`~PyTypeObject.tp_members` from the slot during + class creation, Python clears the flag and sets :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct. .. index:: diff --git a/Doc/c-api/subinterpreters.rst b/Doc/c-api/subinterpreters.rst new file mode 100644 index 000000000000000..83c3fc3d801e9bd --- /dev/null +++ b/Doc/c-api/subinterpreters.rst @@ -0,0 +1,491 @@ +.. highlight:: c + +.. _sub-interpreter-support: + +Multiple interpreters in a Python process +========================================= + +While in most uses, you will only embed a single Python interpreter, there +are cases where you need to create several independent interpreters in the +same process and perhaps even in the same thread. Sub-interpreters allow +you to do that. + +The "main" interpreter is the first one created when the runtime initializes. +It is usually the only Python interpreter in a process. Unlike sub-interpreters, +the main interpreter has unique process-global responsibilities like signal +handling. It is also responsible for execution during runtime initialization and +is usually the active interpreter during runtime finalization. The +:c:func:`PyInterpreterState_Main` function returns a pointer to its state. + +You can switch between sub-interpreters using the :c:func:`PyThreadState_Swap` +function. You can create and destroy them using the following functions: + + +.. c:type:: PyInterpreterConfig + + Structure containing most parameters to configure a sub-interpreter. + Its values are used only in :c:func:`Py_NewInterpreterFromConfig` and + never modified by the runtime. + + .. versionadded:: 3.12 + + Structure fields: + + .. c:member:: int use_main_obmalloc + + If this is ``0`` then the sub-interpreter will use its own + "object" allocator state. + Otherwise it will use (share) the main interpreter's. + + If this is ``0`` then + :c:member:`~PyInterpreterConfig.check_multi_interp_extensions` + must be ``1`` (non-zero). + If this is ``1`` then :c:member:`~PyInterpreterConfig.gil` + must not be :c:macro:`PyInterpreterConfig_OWN_GIL`. + + .. c:member:: int allow_fork + + If this is ``0`` then the runtime will not support forking the + process in any thread where the sub-interpreter is currently active. + Otherwise fork is unrestricted. + + Note that the :mod:`subprocess` module still works + when fork is disallowed. + + .. c:member:: int allow_exec + + If this is ``0`` then the runtime will not support replacing the + current process via exec (e.g. :func:`os.execv`) in any thread + where the sub-interpreter is currently active. + Otherwise exec is unrestricted. + + Note that the :mod:`subprocess` module still works + when exec is disallowed. + + .. c:member:: int allow_threads + + If this is ``0`` then the sub-interpreter's :mod:`threading` module + won't create threads. + Otherwise threads are allowed. + + .. c:member:: int allow_daemon_threads + + If this is ``0`` then the sub-interpreter's :mod:`threading` module + won't create daemon threads. + Otherwise daemon threads are allowed (as long as + :c:member:`~PyInterpreterConfig.allow_threads` is non-zero). + + .. c:member:: int check_multi_interp_extensions + + If this is ``0`` then all extension modules may be imported, + including legacy (single-phase init) modules, + in any thread where the sub-interpreter is currently active. + Otherwise only multi-phase init extension modules + (see :pep:`489`) may be imported. + (Also see :c:macro:`Py_mod_multiple_interpreters`.) + + This must be ``1`` (non-zero) if + :c:member:`~PyInterpreterConfig.use_main_obmalloc` is ``0``. + + .. c:member:: int gil + + This determines the operation of the GIL for the sub-interpreter. + It may be one of the following: + + .. c:namespace:: NULL + + .. c:macro:: PyInterpreterConfig_DEFAULT_GIL + + Use the default selection (:c:macro:`PyInterpreterConfig_SHARED_GIL`). + + .. c:macro:: PyInterpreterConfig_SHARED_GIL + + Use (share) the main interpreter's GIL. + + .. c:macro:: PyInterpreterConfig_OWN_GIL + + Use the sub-interpreter's own GIL. + + If this is :c:macro:`PyInterpreterConfig_OWN_GIL` then + :c:member:`PyInterpreterConfig.use_main_obmalloc` must be ``0``. + + +.. c:function:: PyStatus Py_NewInterpreterFromConfig(PyThreadState **tstate_p, const PyInterpreterConfig *config) + + .. index:: + pair: module; builtins + pair: module; __main__ + pair: module; sys + single: stdout (in module sys) + single: stderr (in module sys) + single: stdin (in module sys) + + Create a new sub-interpreter. This is an (almost) totally separate environment + for the execution of Python code. In particular, the new interpreter has + separate, independent versions of all imported modules, including the + fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. The + table of loaded modules (``sys.modules``) and the module search path + (``sys.path``) are also separate. The new environment has no ``sys.argv`` + variable. It has new standard I/O stream file objects ``sys.stdin``, + ``sys.stdout`` and ``sys.stderr`` (however these refer to the same underlying + file descriptors). + + The given *config* controls the options with which the interpreter + is initialized. + + Upon success, *tstate_p* will be set to the first :term:`thread state` + created in the new sub-interpreter. This thread state is + :term:`attached `. + Note that no actual thread is created; see the discussion of thread states + below. If creation of the new interpreter is unsuccessful, + *tstate_p* is set to ``NULL``; + no exception is set since the exception state is stored in the + :term:`attached thread state`, which might not exist. + + Like all other Python/C API functions, an :term:`attached thread state` + must be present before calling this function, but it might be detached upon + returning. On success, the returned thread state will be :term:`attached `. + If the sub-interpreter is created with its own :term:`GIL` then the + :term:`attached thread state` of the calling interpreter will be detached. + When the function returns, the new interpreter's :term:`thread state` + will be :term:`attached ` to the current thread and + the previous interpreter's :term:`attached thread state` will remain detached. + + .. versionadded:: 3.12 + + Sub-interpreters are most effective when isolated from each other, + with certain functionality restricted:: + + PyInterpreterConfig config = { + .use_main_obmalloc = 0, + .allow_fork = 0, + .allow_exec = 0, + .allow_threads = 1, + .allow_daemon_threads = 0, + .check_multi_interp_extensions = 1, + .gil = PyInterpreterConfig_OWN_GIL, + }; + PyThreadState *tstate = NULL; + PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + + Note that the config is used only briefly and does not get modified. + During initialization the config's values are converted into various + :c:type:`PyInterpreterState` values. A read-only copy of the config + may be stored internally on the :c:type:`PyInterpreterState`. + + .. index:: + single: Py_FinalizeEx (C function) + single: Py_Initialize (C function) + + Extension modules are shared between (sub-)interpreters as follows: + + * For modules using multi-phase initialization, + e.g. :c:func:`PyModule_FromDefAndSpec`, a separate module object is + created and initialized for each interpreter. + Only C-level static and global variables are shared between these + module objects. + + * For modules using legacy + :ref:`single-phase initialization `, + e.g. :c:func:`PyModule_Create`, the first time a particular extension + is imported, it is initialized normally, and a (shallow) copy of its + module's dictionary is squirreled away. + When the same extension is imported by another (sub-)interpreter, a new + module is initialized and filled with the contents of this copy; the + extension's ``init`` function is not called. + Objects in the module's dictionary thus end up shared across + (sub-)interpreters, which might cause unwanted behavior (see + `Bugs and caveats`_ below). + + Note that this is different from what happens when an extension is + imported after the interpreter has been completely re-initialized by + calling :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that + case, the extension's ``initmodule`` function *is* called again. + As with multi-phase initialization, this means that only C-level static + and global variables are shared between these modules. + + .. index:: single: close (in module os) + + +.. c:function:: PyThreadState* Py_NewInterpreter(void) + + .. index:: + pair: module; builtins + pair: module; __main__ + pair: module; sys + single: stdout (in module sys) + single: stderr (in module sys) + single: stdin (in module sys) + + Create a new sub-interpreter. This is essentially just a wrapper + around :c:func:`Py_NewInterpreterFromConfig` with a config that + preserves the existing behavior. The result is an unisolated + sub-interpreter that shares the main interpreter's GIL, allows + fork/exec, allows daemon threads, and allows single-phase init + modules. + + +.. c:function:: void Py_EndInterpreter(PyThreadState *tstate) + + .. index:: single: Py_FinalizeEx (C function) + + Destroy the (sub-)interpreter represented by the given :term:`thread state`. + The given thread state must be :term:`attached `. + When the call returns, there will be no :term:`attached thread state`. + All thread states associated with this interpreter are destroyed. + + :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that + haven't been explicitly destroyed at that point. + + +.. _per-interpreter-gil: + +A per-interpreter GIL +--------------------- + +.. versionadded:: 3.12 + +Using :c:func:`Py_NewInterpreterFromConfig` you can create +a sub-interpreter that is completely isolated from other interpreters, +including having its own GIL. The most important benefit of this +isolation is that such an interpreter can execute Python code without +being blocked by other interpreters or blocking any others. Thus a +single Python process can truly take advantage of multiple CPU cores +when running Python code. The isolation also encourages a different +approach to concurrency than that of just using threads. +(See :pep:`554` and :pep:`684`.) + +Using an isolated interpreter requires vigilance in preserving that +isolation. That especially means not sharing any objects or mutable +state without guarantees about thread-safety. Even objects that are +otherwise immutable (e.g. ``None``, ``(1, 5)``) can't normally be shared +because of the refcount. One simple but less-efficient approach around +this is to use a global lock around all use of some state (or object). +Alternately, effectively immutable objects (like integers or strings) +can be made safe in spite of their refcounts by making them :term:`immortal`. +In fact, this has been done for the builtin singletons, small integers, +and a number of other builtin objects. + +If you preserve isolation then you will have access to proper multi-core +computing without the complications that come with free-threading. +Failure to preserve isolation will expose you to the full consequences +of free-threading, including races and hard-to-debug crashes. + +Aside from that, one of the main challenges of using multiple isolated +interpreters is how to communicate between them safely (not break +isolation) and efficiently. The runtime and stdlib do not provide +any standard approach to this yet. A future stdlib module would help +mitigate the effort of preserving isolation and expose effective tools +for communicating (and sharing) data between interpreters. + + +Bugs and caveats +---------------- + +Because sub-interpreters (and the main interpreter) are part of the same +process, the insulation between them isn't perfect --- for example, using +low-level file operations like :func:`os.close` they can +(accidentally or maliciously) affect each other's open files. Because of the +way extensions are shared between (sub-)interpreters, some extensions may not +work properly; this is especially likely when using single-phase initialization +or (static) global variables. +It is possible to insert objects created in one sub-interpreter into +a namespace of another (sub-)interpreter; this should be avoided if possible. + +Special care should be taken to avoid sharing user-defined functions, +methods, instances or classes between sub-interpreters, since import +operations executed by such objects may affect the wrong (sub-)interpreter's +dictionary of loaded modules. It is equally important to avoid sharing +objects from which the above are reachable. + +Also note that combining this functionality with ``PyGILState_*`` APIs +is delicate, because these APIs assume a bijection between Python thread states +and OS-level threads, an assumption broken by the presence of sub-interpreters. +It is highly recommended that you don't switch sub-interpreters between a pair +of matching :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` calls. +Furthermore, extensions (such as :mod:`ctypes`) using these APIs to allow calling +of Python code from non-Python created threads will probably be broken when using +sub-interpreters. + + +High-level APIs +--------------- + +.. c:type:: PyInterpreterState + + This data structure represents the state shared by a number of cooperating + threads. Threads belonging to the same interpreter share their module + administration and a few other internal items. There are no public members in + this structure. + + Threads belonging to different interpreters initially share nothing, except + process state like available memory, open file descriptors and such. The global + interpreter lock is also shared by all threads, regardless of to which + interpreter they belong. + + .. versionchanged:: 3.12 + + :pep:`684` introduced the possibility + of a :ref:`per-interpreter GIL `. + See :c:func:`Py_NewInterpreterFromConfig`. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Get(void) + + Get the current interpreter. + + Issue a fatal error if there is no :term:`attached thread state`. + It cannot return NULL. + + .. versionadded:: 3.9 + + +.. c:function:: int64_t PyInterpreterState_GetID(PyInterpreterState *interp) + + Return the interpreter's unique ID. If there was any error in doing + so then ``-1`` is returned and an error is set. + + The caller must have an :term:`attached thread state`. + + .. versionadded:: 3.7 + + +.. c:function:: PyObject* PyInterpreterState_GetDict(PyInterpreterState *interp) + + Return a dictionary in which interpreter-specific data may be stored. + If this function returns ``NULL`` then no exception has been raised and + the caller should assume no interpreter-specific dict is available. + + This is not a replacement for :c:func:`PyModule_GetState()`, which + extensions should use to store interpreter-specific state information. + + The returned dictionary is borrowed from the interpreter and is valid until + interpreter shutdown. + + .. versionadded:: 3.8 + + +.. c:type:: PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) + + Type of a frame evaluation function. + + The *throwflag* parameter is used by the ``throw()`` method of generators: + if non-zero, handle the current exception. + + .. versionchanged:: 3.9 + The function now takes a *tstate* parameter. + + .. versionchanged:: 3.11 + The *frame* parameter changed from ``PyFrameObject*`` to ``_PyInterpreterFrame*``. + + +.. c:function:: _PyFrameEvalFunction _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) + + Get the frame evaluation function. + + See the :pep:`523` "Adding a frame evaluation API to CPython". + + .. versionadded:: 3.9 + + +.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) + + Set the frame evaluation function. + + See the :pep:`523` "Adding a frame evaluation API to CPython". + + .. versionadded:: 3.9 + +.. c:function:: void _PyInterpreterState_SetEvalFrameAllowSpecialization(PyInterpreterState *interp, int allow_specialization) + + Enables or disables specialization why a custom frame evaluator is in place. + + If *allow_specialization* is non-zero, the adaptive specializer will + continue to specialize bytecodes even though a custom eval frame function + is set. When *allow_specialization* is zero, setting a custom eval frame + disables specialization. The standard interpreter loop will continue to deopt + while a frame evaluation API is in place - the frame evaluation function needs + to handle the specialized opcodes to take advantage of this. + + .. versionadded:: 3.15 + +.. c:function:: int _PyInterpreterState_IsSpecializationEnabled(PyInterpreterState *interp) + + Return non-zero if adaptive specialization is enabled for the interpreter. + Specialization is enabled when no custom eval frame function is set, or + when one is set with *allow_specialization* enabled. + + .. versionadded:: 3.15 + + +Low-level APIs +-------------- + +All of the following functions must be called after :c:func:`Py_Initialize`. + +.. versionchanged:: 3.7 + :c:func:`Py_Initialize()` now initializes the :term:`GIL` + and sets an :term:`attached thread state`. + + +.. c:function:: PyInterpreterState* PyInterpreterState_New() + + Create a new interpreter state object. An :term:`attached thread state` is not needed, + but may optionally exist if it is necessary to serialize calls to this + function. + + .. audit-event:: cpython.PyInterpreterState_New "" c.PyInterpreterState_New + + +.. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp) + + Reset all information in an interpreter state object. There must be + an :term:`attached thread state` for the interpreter. + + .. audit-event:: cpython.PyInterpreterState_Clear "" c.PyInterpreterState_Clear + + +.. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp) + + Destroy an interpreter state object. There **should not** be an + :term:`attached thread state` for the target interpreter. The interpreter + state must have been reset with a previous call to :c:func:`PyInterpreterState_Clear`. + + +.. _advanced-debugging: + +Advanced debugger support +------------------------- + +These functions are only intended to be used by advanced debugging tools. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Head() + + Return the interpreter state object at the head of the list of all such objects. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Main() + + Return the main interpreter state object. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Next(PyInterpreterState *interp) + + Return the next interpreter state object after *interp* from the list of all + such objects. + + +.. c:function:: PyThreadState * PyInterpreterState_ThreadHead(PyInterpreterState *interp) + + Return the pointer to the first :c:type:`PyThreadState` object in the list of + threads associated with the interpreter *interp*. + + +.. c:function:: PyThreadState* PyThreadState_Next(PyThreadState *tstate) + + Return the next thread state object after *tstate* from the list of all such + objects belonging to the same :c:type:`PyInterpreterState` object. diff --git a/Doc/c-api/synchronization.rst b/Doc/c-api/synchronization.rst new file mode 100644 index 000000000000000..6f18c047a24a92f --- /dev/null +++ b/Doc/c-api/synchronization.rst @@ -0,0 +1,351 @@ +.. highlight:: c + +.. _synchronization: + +Synchronization primitives +========================== + +The C-API provides a basic mutual exclusion lock. + +.. c:type:: PyMutex + + A mutual exclusion lock. The :c:type:`!PyMutex` should be initialized to + zero to represent the unlocked state. For example:: + + PyMutex mutex = {0}; + + Instances of :c:type:`!PyMutex` should not be copied or moved. Both the + contents and address of a :c:type:`!PyMutex` are meaningful, and it must + remain at a fixed, writable location in memory. + + .. note:: + + A :c:type:`!PyMutex` currently occupies one byte, but the size should be + considered unstable. The size may change in future Python releases + without a deprecation period. + + .. versionadded:: 3.13 + +.. c:function:: void PyMutex_Lock(PyMutex *m) + + Lock mutex *m*. If another thread has already locked it, the calling + thread will block until the mutex is unlocked. While blocked, the thread + will temporarily detach the :term:`thread state ` if one exists. + + .. versionadded:: 3.13 + +.. c:function:: void PyMutex_Unlock(PyMutex *m) + + Unlock mutex *m*. The mutex must be locked --- otherwise, the function will + issue a fatal error. + + .. versionadded:: 3.13 + +.. c:function:: int PyMutex_IsLocked(PyMutex *m) + + Returns non-zero if the mutex *m* is currently locked, zero otherwise. + + .. note:: + + This function is intended for use in assertions and debugging only and + should not be used to make concurrency control decisions, as the lock + state may change immediately after the check. + + .. versionadded:: 3.14 + +.. _python-critical-section-api: + +Python critical section API +--------------------------- + +The critical section API provides a deadlock avoidance layer on top of +per-object locks for :term:`free-threaded ` CPython. They are +intended to replace reliance on the :term:`global interpreter lock`, and are +no-ops in versions of Python with the global interpreter lock. + +Critical sections are intended to be used for custom types implemented +in C-API extensions. They should generally not be used with built-in types like +:class:`list` and :class:`dict` because their public C-APIs +already use critical sections internally, with the notable +exception of :c:func:`PyDict_Next`, which requires critical section +to be acquired externally. + +Critical sections avoid deadlocks by implicitly suspending active critical +sections, hence, they do not provide exclusive access such as provided by +traditional locks like :c:type:`PyMutex`. When a critical section is started, +the per-object lock for the object is acquired. If the code executed inside the +critical section calls C-API functions then it can suspend the critical section thereby +releasing the per-object lock, so other threads can acquire the per-object lock +for the same object. + +Variants that accept :c:type:`PyMutex` pointers rather than Python objects are also +available. Use these variants to start a critical section in a situation where +there is no :c:type:`PyObject` -- for example, when working with a C type that +does not extend or wrap :c:type:`PyObject` but still needs to call into the C +API in a manner that might lead to deadlocks. + +.. note:: + + Operations that need to lock two objects at once must use + :c:macro:`Py_BEGIN_CRITICAL_SECTION2`. You *cannot* use nested critical + sections to lock more than one object at once, because the inner critical + section may suspend the outer critical sections. This API does not provide + a way to lock more than two objects at once. + +Example usage:: + + static PyObject * + set_field(MyObject *self, PyObject *value) + { + Py_BEGIN_CRITICAL_SECTION(self); + Py_SETREF(self->field, Py_XNewRef(value)); + Py_END_CRITICAL_SECTION(); + Py_RETURN_NONE; + } + +In the above example, :c:macro:`Py_SETREF` calls :c:macro:`Py_DECREF`, which +can call arbitrary code through an object's deallocation function. The critical +section API avoids potential deadlocks due to reentrancy and lock ordering +by allowing the runtime to temporarily suspend the critical section if the +code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. + +.. _critical-section-macros: + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION(op) + + Acquires the per-object lock for the object *op* and begins a + critical section. + + In the free-threaded build, and when building for the + :ref:`Stable ABI `, this macro expands to:: + + { + PyCriticalSection _py_cs; + PyCriticalSection_Begin(&_py_cs, (PyObject*)(op)) + + In the default build, this macro expands to ``{``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION_MUTEX(m) + + Locks the mutex *m* and begins a critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection _py_cs; + PyCriticalSection_BeginMutex(&_py_cs, m) + + Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION`, there is no cast for + the argument of the macro - it must be a :c:type:`PyMutex` pointer. + + On the default build, this macro expands to ``{``. + + .. versionadded:: 3.14 + +.. c:macro:: Py_END_CRITICAL_SECTION() + + Ends the critical section and releases the per-object lock. + + In the free-threaded build, and when building for the + :ref:`Stable ABI `, this macro expands to:: + + PyCriticalSection_End(&_py_cs); + } + + In the default build, this macro expands to ``}``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION2(a, b) + + Acquires the per-object locks for the objects *a* and *b* and begins a + critical section. The locks are acquired in a consistent order (lowest + address first) to avoid lock ordering deadlocks. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection2 _py_cs2; + PyCriticalSection2_Begin(&_py_cs2, (PyObject*)(a), (PyObject*)(b)) + + In the default build, this macro expands to ``{``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) + + Locks the mutexes *m1* and *m2* and begins a critical section. + + In the free-threaded build, and when building for the + :ref:`Stable ABI `, this macro expands to:: + + { + PyCriticalSection2 _py_cs2; + PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) + + Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION2`, there is no cast for + the arguments of the macro - they must be :c:type:`PyMutex` pointers. + + On the default build, this macro expands to ``{``. + + .. versionadded:: 3.14 + +.. c:macro:: Py_END_CRITICAL_SECTION2() + + Ends the critical section and releases the per-object locks. + + In the free-threaded build, and when building for the + :ref:`Stable ABI `, this macro expands to:: + + PyCriticalSection2_End(&_py_cs2); + } + + In the default build, this macro expands to ``}``. + + .. versionadded:: 3.13 + +Low-level critical section API +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The following functions and structs are exposed for cases where C macros +are not available. + +.. c:function:: void PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) + void PyCriticalSection_End(PyCriticalSection *c) + void PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) + void PyCriticalSection2_End(PyCriticalSection2 *c); + + To be used only as in the macro expansions + listed :ref:`earlier in this section `. + + In non-:term:`free-threaded ` builds of CPython, these + functions do nothing. + + .. versionadded:: 3.13 + +.. c:type:: PyCriticalSection + PyCriticalSection2 + + To be used only as in the macro expansions + listed :ref:`earlier in this section `. + Note that the contents of the structures are private and their meaning may + change in future Python versions. + + .. versionadded:: 3.13 + +.. c:function:: void PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m); + void PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2); + + .. (These need to be in a separate section without a Stable ABI annotation.) + + To be used only as in the macro expansions + listed :ref:`earlier in this section `. + + In non-:term:`free-threaded ` builds of CPython, these + functions do nothing. + + .. versionadded:: 3.14 + + +Legacy locking APIs +------------------- + +These APIs are obsolete since Python 3.13 with the introduction of +:c:type:`PyMutex`. + +.. versionchanged:: 3.15 + These APIs are now a simple wrapper around ``PyMutex``. + + +.. c:type:: PyThread_type_lock + + A pointer to a mutual exclusion lock. + + +.. c:type:: PyLockStatus + + The result of acquiring a lock with a timeout. + + .. c:namespace:: NULL + + .. c:enumerator:: PY_LOCK_FAILURE + + Failed to acquire the lock. + + .. c:enumerator:: PY_LOCK_ACQUIRED + + The lock was successfully acquired. + + .. c:enumerator:: PY_LOCK_INTR + + The lock was interrupted by a signal. + + +.. c:function:: PyThread_type_lock PyThread_allocate_lock(void) + + Allocate a new lock. + + On success, this function returns a lock; on failure, this + function returns ``0`` without an exception set. + + The caller does not need to hold an :term:`attached thread state`. + + .. versionchanged:: 3.15 + This function now always uses :c:type:`PyMutex`. In prior versions, this + would use a lock provided by the operating system. + + +.. c:function:: void PyThread_free_lock(PyThread_type_lock lock) + + Destroy *lock*. The lock should not be held by any thread when calling + this. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: PyLockStatus PyThread_acquire_lock_timed(PyThread_type_lock lock, long long microseconds, int intr_flag) + + Acquire *lock* with a timeout. + + This will wait for *microseconds* microseconds to acquire the lock. If the + timeout expires, this function returns :c:enumerator:`PY_LOCK_FAILURE`. + If *microseconds* is ``-1``, this will wait indefinitely until the lock has + been released. + + If *intr_flag* is ``1``, acquiring the lock may be interrupted by a signal, + in which case this function returns :c:enumerator:`PY_LOCK_INTR`. Upon + interruption, it's generally expected that the caller makes a call to + :c:func:`Py_MakePendingCalls` to propagate an exception to Python code. + + If the lock is successfully acquired, this function returns + :c:enumerator:`PY_LOCK_ACQUIRED`. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: int PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) + + Acquire *lock*. + + If *waitflag* is ``1`` and another thread currently holds the lock, this + function will wait until the lock can be acquired and will always return + ``1``. + + If *waitflag* is ``0`` and another thread holds the lock, this function will + not wait and instead return ``0``. If the lock is not held by any other + thread, then this function will acquire it and return ``1``. + + Unlike :c:func:`PyThread_acquire_lock_timed`, acquiring the lock cannot be + interrupted by a signal. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: int PyThread_release_lock(PyThread_type_lock lock) + + Release *lock*. If *lock* is not held, then this function issues a + fatal error. + + The caller does not need to hold an :term:`attached thread state`. diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index b34936dd55e94ca..ee73c1c8adaa7b3 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -123,6 +123,24 @@ Operating System Utilities This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions directly! + +.. c:function:: int PyOS_InterruptOccurred(void) + + Check if a :c:macro:`!SIGINT` signal has been received. + + Returns ``1`` if a :c:macro:`!SIGINT` has occurred and clears the signal flag, + or ``0`` otherwise. + + In most cases, you should prefer :c:func:`PyErr_CheckSignals` over this function. + :c:func:`!PyErr_CheckSignals` invokes the appropriate signal handlers + for all pending signals, allowing Python code to handle the signal properly. + This function only detects :c:macro:`!SIGINT` and does not invoke any Python + signal handlers. + + This function is async-signal-safe and this function cannot fail. + The caller must hold an :term:`attached thread state`. + + .. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size) .. warning:: @@ -268,7 +286,7 @@ accessible to C code. They all work with the current interpreter thread's If the non-existing object should not be treated as a failure, you can use :c:func:`PySys_GetOptionalAttr` instead. - .. versionadded:: next + .. versionadded:: 3.15 .. c:function:: PyObject *PySys_GetAttrString(const char *name) @@ -279,7 +297,7 @@ accessible to C code. They all work with the current interpreter thread's If the non-existing object should not be treated as a failure, you can use :c:func:`PySys_GetOptionalAttrString` instead. - .. versionadded:: next + .. versionadded:: 3.15 .. c:function:: int PySys_GetOptionalAttr(PyObject *name, PyObject **result) @@ -293,7 +311,7 @@ accessible to C code. They all work with the current interpreter thread's * Set an exception, set *\*result* to ``NULL``, and return ``-1``, if an error occurred. - .. versionadded:: next + .. versionadded:: 3.15 .. c:function:: int PySys_GetOptionalAttrString(const char *name, PyObject **result) @@ -301,7 +319,7 @@ accessible to C code. They all work with the current interpreter thread's specified as a :c:expr:`const char*` UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. - .. versionadded:: next + .. versionadded:: 3.15 .. c:function:: PyObject *PySys_GetObject(const char *name) @@ -316,14 +334,6 @@ accessible to C code. They all work with the current interpreter thread's case *name* is deleted from the sys module. Returns ``0`` on success, ``-1`` on error. -.. c:function:: void PySys_ResetWarnOptions() - - Reset :data:`sys.warnoptions` to an empty list. This function may be - called prior to :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. - .. c:function:: void PySys_WriteStdout(const char *format, ...) Write the output string described by *format* to :data:`sys.stdout`. No diff --git a/Doc/c-api/threads.rst b/Doc/c-api/threads.rst new file mode 100644 index 000000000000000..ca34abd73d8423b --- /dev/null +++ b/Doc/c-api/threads.rst @@ -0,0 +1,994 @@ +.. highlight:: c + +.. _threads: + +Thread states and the global interpreter lock +============================================= + +.. index:: + single: global interpreter lock + single: interpreter lock + single: lock, interpreter + +Unless on a :term:`free-threaded build` of :term:`CPython`, +the Python interpreter is generally not thread-safe. In order to support +multi-threaded Python programs, there's a global lock, called the :term:`global +interpreter lock` or :term:`GIL`, that must be held by a thread before +accessing Python objects. Without the lock, even the simplest operations +could cause problems in a multi-threaded program: for example, when +two threads simultaneously increment the reference count of the same object, the +reference count could end up being incremented only once instead of twice. + +As such, only a thread that holds the GIL may operate on Python objects or +invoke Python's C API. + +.. index:: single: setswitchinterval (in module sys) + +In order to emulate concurrency, the interpreter regularly tries to switch +threads between bytecode instructions (see :func:`sys.setswitchinterval`). +This is why locks are also necessary for thread-safety in pure-Python code. + +Additionally, the global interpreter lock is released around blocking I/O +operations, such as reading or writing to a file. From the C API, this is done +by :ref:`detaching the thread state `. + + +.. index:: + single: PyThreadState (C type) + +The Python interpreter keeps some thread-local information inside +a data structure called :c:type:`PyThreadState`, known as a :term:`thread state`. +Each thread has a thread-local pointer to a :c:type:`PyThreadState`; a thread state +referenced by this pointer is considered to be :term:`attached `. + +A thread can only have one :term:`attached thread state` at a time. An attached +thread state is typically analogous with holding the GIL, except on +free-threaded builds. On builds with the GIL enabled, attaching a thread state +will block until the GIL can be acquired. However, even on builds with the GIL +disabled, it is still required to have an attached thread state, as the interpreter +needs to keep track of which threads may access Python objects. + +.. note:: + + Even on the free-threaded build, attaching a thread state may block, as the + GIL can be re-enabled or threads might be temporarily suspended (such as during + a garbage collection). + +Generally, there will always be an attached thread state when using Python's +C API, including during embedding and when implementing methods, so it's uncommon +to need to set up a thread state on your own. Only in some specific cases, such +as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block or in a fresh thread, will the +thread not have an attached thread state. +If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns ``NULL``. + +If it turns out that you do need to create a thread state, it is recommended to +use :c:func:`PyThreadState_Ensure` or :c:func:`PyThreadState_EnsureFromView`, +which will manage the thread state for you. + + +.. _detaching-thread-state: + +Detaching the thread state from extension code +---------------------------------------------- + +Most extension code manipulating the :term:`thread state` has the following simple +structure:: + + Save the thread state in a local variable. + ... Do some blocking I/O operation ... + Restore the thread state from the local variable. + +This is so common that a pair of macros exists to simplify it:: + + Py_BEGIN_ALLOW_THREADS + ... Do some blocking I/O operation ... + Py_END_ALLOW_THREADS + +.. index:: + single: Py_BEGIN_ALLOW_THREADS (C macro) + single: Py_END_ALLOW_THREADS (C macro) + +The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a +hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the +block. + +The block above expands to the following code:: + + PyThreadState *_save; + + _save = PyEval_SaveThread(); + ... Do some blocking I/O operation ... + PyEval_RestoreThread(_save); + +.. index:: + single: PyEval_RestoreThread (C function) + single: PyEval_SaveThread (C function) + +Here is how these functions work: + +The attached thread state implies that the GIL is held for the interpreter. +To detach it, :c:func:`PyEval_SaveThread` is called and the result is stored +in a local variable. + +By detaching the thread state, the GIL is released, which allows other threads +to attach to the interpreter and execute while the current thread performs +blocking I/O. When the I/O operation is complete, the old thread state is +reattached by calling :c:func:`PyEval_RestoreThread`, which will wait until +the GIL can be acquired. + +.. note:: + Performing blocking I/O is the most common use case for detaching + the thread state, but it is also useful to call it over long-running + native code that doesn't need access to Python objects or Python's C API. + For example, the standard :mod:`zlib` and :mod:`hashlib` modules detach the + :term:`thread state ` when compressing or hashing + data. + +On a :term:`free-threaded build`, the :term:`GIL` is usually out of the question, +but **detaching the thread state is still required**, because the interpreter +periodically needs to block all threads to get a consistent view of Python objects +without the risk of race conditions. +For example, CPython currently suspends all threads for a short period of time +while running the garbage collector. + +.. warning:: + + Detaching the thread state can lead to unexpected behavior during interpreter + finalization. See :ref:`cautions-regarding-runtime-finalization` for more + details. + + +APIs +^^^^ + +The following macros are normally used without a trailing semicolon; look for +example usage in the Python source distribution. + +.. note:: + + These macros are still necessary on the :term:`free-threaded build` to prevent + deadlocks. + +.. c:macro:: Py_BEGIN_ALLOW_THREADS + + This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. + Note that it contains an opening brace; it must be matched with a following + :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this + macro. + + +.. c:macro:: Py_END_ALLOW_THREADS + + This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains + a closing brace; it must be matched with an earlier + :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of + this macro. + + +.. c:macro:: Py_BLOCK_THREADS + + This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to + :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. + + +.. c:macro:: Py_UNBLOCK_THREADS + + This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to + :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable + declaration. + + +.. _non-python-created-threads: +.. _c-api-foreign-threads: + + +Using the C API from foreign threads +------------------------------------ + +When threads are created using the dedicated Python APIs (such as the +:mod:`threading` module), a thread state is automatically associated with them, +However, when a thread is created from native code (for example, by a +third-party library with its own thread management), it doesn't hold an +attached thread state. + +If you need to call Python code from these threads (often this will be part +of a callback API provided by the aforementioned third-party library), +you must first register these threads with the interpreter by +creating a new thread state and attaching it. + +The easiest way to do this is through :c:func:`PyThreadState_Ensure` +or :c:func:`PyThreadState_EnsureFromView`. + +.. note:: + These functions require an argument pointing to the desired + interpreter; such a pointer can be acquired via a call to + :c:func:`PyInterpreterGuard_FromCurrent` (for ``PyThreadState_Ensure``) or + :c:func:`PyInterpreterView_FromCurrent` (for ``PyThreadState_EnsureFromView``) + from the function that creates the thread. If no pointer is available (such + as when the given native thread library doesn't provide a data argument), + :c:func:`PyInterpreterView_FromMain` can be used to get a view for the main + interpreter, but note that this will make the code incompatible with + subinterpreters. + + +For example:: + + // The return value of PyInterpreterGuard_FromCurrent() from the + // function that created this thread. + PyInterpreterGuard *guard = thread_data->guard; + + // Create a new thread state for the interpreter. + PyThreadStateToken *token = PyThreadState_Ensure(guard); + if (token == NULL) { + PyInterpreterGuard_Close(guard); + return; + } + + // We have a valid thread state -- perform Python actions here. + result = CallSomeFunction(); + // Evaluate result or handle exceptions. + + // Release the thread state. No calls to the C API are allowed beyond this + // point. + PyThreadState_Release(token); + PyInterpreterGuard_Close(guard); + + +Keep in mind that calling ``PyThreadState_Ensure`` might not always create a new +thread state, and calling ``PyThreadState_Release`` might not always detach it. +These functions may reuse an existing attached thread state, or may re-attach +a thread state that was previously attached for the current thread. + +.. seealso:: + :pep:`788` + +.. _c-api-attach-detach: + +Attaching/detaching thread states +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: PyThreadStateToken *PyThreadState_Ensure(PyInterpreterGuard *guard) + + Ensure that the thread has an attached thread state for the + interpreter protected by *guard*, and thus can safely invoke that + interpreter. + + It is OK to call this function if the thread already has an + attached thread state, as long as there is a subsequent call to + :c:func:`PyThreadState_Release` that matches this one (meaning that "nested" + calls to this function are permitted). + + The function's effect (if any) will be reversed by the matching call to + :c:func:`PyThreadState_Release`. + + On error, this function returns ``NULL`` *without* an exception set. + Do not call :c:func:`!PyThreadState_Release` in this case. + + On success, this function returns a pointer value that must be passed + to the matching call to :c:func:`!PyThreadState_Release`. + + The conditions in which this function creates a new :term:`thread state` are + considered unstable and implementation-dependent. If you need to control the + exact lifetime of a thread state, consider using :c:func:`PyThreadState_New`. + However, do not avoid this function solely on the basis that the lifetime + of the thread state may be inconsistent across versions; changes to this + function will be done with caution and in a backwards-compatible manner. + In particular, the saving of thread-local variables and similar state will + be retained across Python versions. + + .. impl-detail:: + + The exact behavior of whether this function creates a new thread state is + described below, but be aware that this may change in the future. + + First, this function checks if an attached thread state is present. + If there is, this function then checks if the interpreter of that + thread state matches the interpreter guarded by *guard*. If that is + the case, this function simply marks the thread state as being used + by a ``PyThreadState_Ensure`` call and returns. + + If there is no attached thread state, then this function checks if any + thread state has been used by the current OS thread. (This is + returned by :c:func:`PyGILState_GetThisThreadState`.) + If there was, then this function checks if that thread state's interpreter + matches *guard*. If it does, it is re-attached and marked as used. + + Otherwise, if both of the above cases fail, a new thread state is created + for *guard*. It is then attached and marked as owned by ``PyThreadState_Ensure``. + + .. versionadded:: 3.15 + + +.. c:function:: PyThreadStateToken *PyThreadState_EnsureFromView(PyInterpreterView *view) + + Get an attached thread state for the interpreter referenced by *view*. + + The behavior and return value are the same as for :c:func:`PyThreadState_Ensure`; + additionally, if the function succeeds, the interpreter referenced by *view* will + be implicitly guarded. The guard will be released upon the corresponding + :c:func:`PyThreadState_Release` call. + + .. versionadded:: 3.15 + + +.. c:function:: void PyThreadState_Release(PyThreadStateToken *token) + + Undo a :c:func:`PyThreadState_Ensure` or + :c:func:`PyThreadState_EnsureFromView` call. + + This must be called exactly once for each successful *Ensure* call, with + *token* set to that call's return value. + + The state that was attached before the corresponding *Ensure* call + (if any) will be attached when :c:func:`PyThreadState_Release` returns. + + The exact behavior of whether this function deletes a thread state is + considered unstable and implementation-dependent. + + .. impl-detail:: + + Currently, this function will decrement an internal counter on the + attached thread state. If this counter ever reaches below zero, this + function emits a fatal error (via :c:func:`Py_FatalError`). + + If the attached thread state is owned by ``PyThreadState_Ensure``, then the + attached thread state will be deallocated and deleted upon the internal counter + reaching zero. Otherwise, nothing happens when the counter reaches zero. + + .. versionadded:: 3.15 + +.. c:type:: PyThreadStateToken + + An opaque token retrieved from a :c:func:`PyThreadState_Ensure` call + and passed to a corresponding :c:func:`PyThreadState_Release` call. + + +.. _legacy-api: +.. _gilstate: + +GIL-state APIs +-------------- + +The following APIs are generally not compatible with subinterpreters and +will hang the process during interpreter finalization (see +:ref:`cautions-regarding-runtime-finalization`). As such, these APIs were +:term:`soft deprecated` in Python 3.15 in favor of the :ref:`new APIs +`. + + +.. c:type:: PyGILState_STATE + + The type of the value returned by :c:func:`PyGILState_Ensure` and passed to + :c:func:`PyGILState_Release`. + + .. c:enumerator:: PyGILState_LOCKED + + The GIL was already held when :c:func:`PyGILState_Ensure` was called. + + .. c:enumerator:: PyGILState_UNLOCKED + + The GIL was not held when :c:func:`PyGILState_Ensure` was called. + + +.. c:function:: PyGILState_STATE PyGILState_Ensure() + + Ensure that the current thread is ready to call the Python C API regardless + of the current state of Python, or of the :term:`attached thread state`. This may + be called as many times as desired by a thread as long as each call is + matched with a call to :c:func:`PyGILState_Release`. In general, other + thread-related APIs may be used between :c:func:`PyGILState_Ensure` and + :c:func:`PyGILState_Release` calls as long as the thread state is restored to + its previous state before the Release(). For example, normal usage of the + :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is + acceptable. + + The return value is an opaque "handle" to the :term:`attached thread state` when + :c:func:`PyGILState_Ensure` was called, and must be passed to + :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even + though recursive calls are allowed, these handles *cannot* be shared - each + unique call to :c:func:`PyGILState_Ensure` must save the handle for its call + to :c:func:`PyGILState_Release`. + + When the function returns, there will be an :term:`attached thread state` + and the thread will be able to call arbitrary Python code. + + This function has no way to return an error. As such, errors are either fatal + (that is, they send ``SIGABRT`` and crash the process; see + :c:func:`Py_FatalError`), or the thread will be permanently blocked (such as + during interpreter finalization). + + .. warning:: + Calling this function when the interpreter is finalizing will + infinitely hang the thread, which may cause deadlocks. + :ref:`cautions-regarding-runtime-finalization` for more details. + + In addition, this function generally does not work with subinterpreters + when used from foreign threads, because this function has no way of + knowing which interpreter created the thread (and as such, will implicitly + pick the main interpreter). + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Ensure` or + :c:func:`PyThreadState_EnsureFromView` instead. + + +.. c:function:: void PyGILState_Release(PyGILState_STATE) + + Release any resources previously acquired. After this call, Python's state will + be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call + (but generally this state will be unknown to the caller, hence the use of the + GIL-state API). + + Every call to :c:func:`PyGILState_Ensure` must be matched by a call to + :c:func:`PyGILState_Release` on the same thread. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Release` instead. + + +.. c:function:: PyThreadState* PyGILState_GetThisThreadState() + + Get the :term:`thread state` that was most recently :term:`attached + ` for this thread. (If the most recent thread state + has been deleted, this returns ``NULL``.) + + If the caller has an attached thread state, it is returned. + + In other terms, this function returns the thread state that will be used by + :c:func:`PyGILState_Ensure`. If this returns ``NULL``, then + ``PyGILState_Ensure`` will create a new thread state. + + This function cannot fail. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` + instead. + + +.. c:function:: int PyGILState_Check() + + Return ``1`` if the current thread has an :term:`attached thread state` + that matches the thread state returned by + :c:func:`PyGILState_GetThisThreadState`. If the caller has no attached thread + state or it otherwise doesn't match, then this returns ``0``. + + If the current Python process has ever created a subinterpreter, this + function will *always* return ``1``. + + This is mainly a helper/diagnostic function. + + .. versionadded:: 3.4 + + .. soft-deprecated:: 3.15 + Use ``PyThreadState_GetUnchecked() != NULL`` instead. + + +.. _fork-and-threads: + +Cautions about fork() +--------------------- + +Another important thing to note about threads is their behaviour in the face +of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a +process forks only the thread that issued the fork will exist. This has a +concrete impact both on how locks must be handled and on all stored state +in CPython's runtime. + +The fact that only the "current" thread remains +means any locks held by other threads will never be released. Python solves +this for :func:`os.fork` by acquiring the locks it uses internally before +the fork, and releasing them afterwards. In addition, it resets any +:ref:`lock-objects` in the child. When extending or embedding Python, there +is no way to inform Python of additional (non-Python) locks that need to be +acquired before or reset after a fork. OS facilities such as +:c:func:`!pthread_atfork` would need to be used to accomplish the same thing. +Additionally, when extending or embedding Python, calling :c:func:`fork` +directly rather than through :func:`os.fork` (and returning to or calling +into Python) may result in a deadlock by one of Python's internal locks +being held by a thread that is defunct after the fork. +:c:func:`PyOS_AfterFork_Child` tries to reset the necessary locks, but is not +always able to. + +The fact that all other threads go away also means that CPython's +runtime state there must be cleaned up properly, which :func:`os.fork` +does. This means finalizing all other :c:type:`PyThreadState` objects +belonging to the current interpreter and all other +:c:type:`PyInterpreterState` objects. Due to this and the special +nature of the :ref:`"main" interpreter `, +:c:func:`fork` should only be called in that interpreter's "main" +thread, where the CPython global runtime was originally initialized. +The only exception is if :c:func:`exec` will be called immediately +after. + + +High-level APIs +--------------- + +These are the most commonly used types and functions when writing multi-threaded +C extensions. + + +.. c:type:: PyThreadState + + This data structure represents the state of a single thread. The only public + data member is: + + .. c:member:: PyInterpreterState *interp + + This thread's interpreter state. + + +.. c:function:: void PyEval_InitThreads() + + .. index:: + single: PyEval_AcquireThread() + single: PyEval_ReleaseThread() + single: PyEval_SaveThread() + single: PyEval_RestoreThread() + + Deprecated function which does nothing. + + In Python 3.6 and older, this function created the GIL if it didn't exist. + + .. versionchanged:: 3.9 + The function now does nothing. + + .. versionchanged:: 3.7 + This function is now called by :c:func:`Py_Initialize()`, so you don't + have to call it yourself anymore. + + .. versionchanged:: 3.2 + This function cannot be called before :c:func:`Py_Initialize()` anymore. + + .. deprecated:: 3.9 + + .. index:: pair: module; _thread + + +.. c:function:: PyThreadState* PyEval_SaveThread() + + Detach the :term:`attached thread state` and return it. + The thread will have no :term:`thread state` upon returning. + + +.. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) + + Set the :term:`attached thread state` to *tstate*. + The passed :term:`thread state` **should not** be :term:`attached `, + otherwise deadlock ensues. *tstate* will be attached upon returning. + + .. note:: + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + +.. c:function:: PyThreadState* PyThreadState_Get() + + Return the :term:`attached thread state`. If the thread has no attached + thread state, (such as when inside of :c:macro:`Py_BEGIN_ALLOW_THREADS` + block), then this issues a fatal error (so that the caller needn't check + for ``NULL``). + + See also :c:func:`PyThreadState_GetUnchecked`. + +.. c:function:: PyThreadState* PyThreadState_GetUnchecked() + + Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a + fatal error if it is NULL. The caller is responsible to check if the result + is NULL. + + .. versionadded:: 3.13 + In Python 3.5 to 3.12, the function was private and known as + ``_PyThreadState_UncheckedGet()``. + + +.. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) + + Set the :term:`attached thread state` to *tstate*, and return the + :term:`thread state` that was attached prior to calling. + + This function is safe to call without an :term:`attached thread state`; it + will simply return ``NULL`` indicating that there was no prior thread state. + + .. seealso:: + :c:func:`PyEval_ReleaseThread` + + .. note:: + Similar to :c:func:`PyGILState_Ensure`, this function will hang the + thread if the runtime is finalizing. + + +Low-level APIs +-------------- + +.. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) + + Create a new thread state object belonging to the given interpreter object. + An :term:`attached thread state` is not needed. + +.. c:function:: void PyThreadState_Clear(PyThreadState *tstate) + + Reset all information in a :term:`thread state` object. *tstate* + must be :term:`attached ` + + .. versionchanged:: 3.9 + This function now calls the :c:member:`!PyThreadState.on_delete` callback. + Previously, that happened in :c:func:`PyThreadState_Delete`. + + .. versionchanged:: 3.13 + The :c:member:`!PyThreadState.on_delete` callback was removed. + + +.. c:function:: void PyThreadState_Delete(PyThreadState *tstate) + + Destroy a :term:`thread state` object. *tstate* should not + be :term:`attached ` to any thread. + *tstate* must have been reset with a previous call to + :c:func:`PyThreadState_Clear`. + + +.. c:function:: void PyThreadState_DeleteCurrent(void) + + Detach the :term:`attached thread state` (which must have been reset + with a previous call to :c:func:`PyThreadState_Clear`) and then destroy it. + + No :term:`thread state` will be :term:`attached ` upon + returning. + +.. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) + + Get the current frame of the Python thread state *tstate*. + + Return a :term:`strong reference`. Return ``NULL`` if no frame is currently + executing. + + See also :c:func:`PyEval_GetFrame`. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: uint64_t PyThreadState_GetID(PyThreadState *tstate) + + Get the unique :term:`thread state` identifier of the Python thread state *tstate*. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: PyInterpreterState* PyThreadState_GetInterpreter(PyThreadState *tstate) + + Get the interpreter of the Python thread state *tstate*. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: void PyThreadState_EnterTracing(PyThreadState *tstate) + + Suspend tracing and profiling in the Python thread state *tstate*. + + Resume them using the :c:func:`PyThreadState_LeaveTracing` function. + + .. versionadded:: 3.11 + + +.. c:function:: void PyThreadState_LeaveTracing(PyThreadState *tstate) + + Resume tracing and profiling in the Python thread state *tstate* suspended + by the :c:func:`PyThreadState_EnterTracing` function. + + See also :c:func:`PyEval_SetTrace` and :c:func:`PyEval_SetProfile` + functions. + + .. versionadded:: 3.11 + + +.. c:function:: int PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, void *stack_start_addr, size_t stack_size) + + Set the stack protection start address and stack protection size + of a Python thread state. + + On success, return ``0``. + On failure, set an exception and return ``-1``. + + CPython implements :ref:`recursion control ` for C code by raising + :py:exc:`RecursionError` when it notices that the machine execution stack is close + to overflow. See for example the :c:func:`Py_EnterRecursiveCall` function. + For this, it needs to know the location of the current thread's stack, which it + normally gets from the operating system. + When the stack is changed, for example using context switching techniques like the + Boost library's ``boost::context``, you must call + :c:func:`~PyUnstable_ThreadState_SetStackProtection` to inform CPython of the change. + + Call :c:func:`~PyUnstable_ThreadState_SetStackProtection` either before + or after changing the stack. + Do not call any other Python C API between the call and the stack + change. + + See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. + + .. versionadded:: 3.15 + + +.. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) + + Reset the stack protection start address and stack protection size + of a Python thread state to the operating system defaults. + + See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. + + .. versionadded:: 3.15 + + +.. c:function:: PyObject* PyThreadState_GetDict() + + Return a dictionary in which extensions can store thread-specific state + information. Each extension should use a unique key to store a state in + the dictionary. It is okay to call this function when no :term:`thread state` + is :term:`attached `. If this function returns + ``NULL`` and no exception has been raised, then the caller should assume no + thread state is attached. + + +.. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) + + :term:`Attach ` *tstate* to the current thread, + which must not be ``NULL`` or already :term:`attached `. + + The calling thread must not already have an :term:`attached thread state`. + + .. note:: + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.8 + Updated to be consistent with :c:func:`PyEval_RestoreThread`, + :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, + and terminate the current thread if called while the interpreter is finalizing. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + + :c:func:`PyEval_RestoreThread` is a higher-level function which is always + available (even when threads have not been initialized). + + +.. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) + + Detach the :term:`attached thread state`. + The *tstate* argument, which must not be ``NULL``, is only used to check + that it represents the :term:`attached thread state` --- if it isn't, a fatal error is + reported. + + :c:func:`PyEval_SaveThread` is a higher-level function which is always + available (even when threads have not been initialized). + + +Asynchronous notifications +========================== + +A mechanism is provided to make asynchronous notifications to the main +interpreter thread. These notifications take the form of a function +pointer and a void pointer argument. + + +.. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) + + Schedule a function to be called from the main interpreter thread. On + success, ``0`` is returned and *func* is queued for being called in the + main thread. On failure, ``-1`` is returned without setting any exception. + + When successfully queued, *func* will be *eventually* called from the + main interpreter thread with the argument *arg*. It will be called + asynchronously with respect to normally running Python code, but with + both these conditions met: + + * on a :term:`bytecode` boundary; + * with the main thread holding an :term:`attached thread state` + (*func* can therefore use the full C API). + + *func* must return ``0`` on success, or ``-1`` on failure with an exception + set. *func* won't be interrupted to perform another asynchronous + notification recursively, but it can still be interrupted to switch + threads if the :term:`thread state ` is detached. + + This function doesn't need an :term:`attached thread state`. However, to call this + function in a subinterpreter, the caller must have an :term:`attached thread state`. + Otherwise, the function *func* can be scheduled to be called from the wrong interpreter. + + .. warning:: + This is a low-level function, only useful for very special cases. + There is no guarantee that *func* will be called as quick as + possible. If the main thread is busy executing a system call, + *func* won't be called before the system call returns. This + function is generally **not** suitable for calling Python code from + arbitrary C threads. Instead, use :c:func:`PyThreadState_EnsureFromView`. + + .. versionadded:: 3.1 + + .. versionchanged:: 3.9 + If this function is called in a subinterpreter, the function *func* is + now scheduled to be called from the subinterpreter, rather than being + called from the main interpreter. Each subinterpreter now has its own + list of scheduled calls. + + .. versionchanged:: 3.12 + This function now always schedules *func* to be run in the main + interpreter. + + +.. c:function:: int Py_MakePendingCalls(void) + + Execute all pending calls. This is usually executed automatically by the + interpreter. + + This function returns ``0`` on success, and returns ``-1`` with an exception + set on failure. + + If this is not called in the main thread of the main + interpreter, this function does nothing and returns ``0``. + The caller must hold an :term:`attached thread state`. + + .. versionadded:: 3.1 + + .. versionchanged:: 3.12 + This function only runs pending calls in the main interpreter. + + +.. c:function:: int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) + + Schedule an exception to be raised asynchronously in a thread. + If the thread has a previously scheduled exception, it is overwritten. + + The *id* argument is the thread id of the target thread, as returned by + :c:func:`PyThread_get_thread_ident`. + *exc* is the class of the exception to be raised, or ``NULL`` to clear + the pending exception (if any). + + Return the number of affected thread states. + This is normally ``1`` if *id* is found, even when no change was + made (the given *exc* was already pending, or *exc* is ``NULL`` but + no exception is pending). + If the thread id isn't found, return ``0``. This raises no exceptions. + + To prevent naive misuse, you must write your own C extension to call this. + This function must be called with an :term:`attached thread state`. + This function does not steal any references to *exc*. + This function does not necessarily interrupt system calls such as + :py:func:`~time.sleep`. + + .. versionchanged:: 3.7 + The type of the *id* parameter changed from :c:expr:`long` to + :c:expr:`unsigned long`. + + +Operating system thread APIs +============================ + +.. c:macro:: PYTHREAD_INVALID_THREAD_ID + + Sentinel value for an invalid thread ID. + + This is currently equivalent to ``(unsigned long)-1``. + + +.. c:function:: unsigned long PyThread_start_new_thread(void (*func)(void *), void *arg) + + Start function *func* in a new thread with argument *arg*. + The resulting thread is not intended to be joined. + + *func* must not be ``NULL``, but *arg* may be ``NULL``. + + On success, this function returns the identifier of the new thread; on failure, + this returns :c:macro:`PYTHREAD_INVALID_THREAD_ID`. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: unsigned long PyThread_get_thread_ident(void) + + Return the identifier of the current thread, which will never be zero. + + This function cannot fail, and the caller does not need to hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`threading.get_ident` and :py:attr:`threading.Thread.ident` + expose this identifier to Python. + + +.. c:function:: PyObject *PyThread_GetInfo(void) + + Get general information about the current thread in the form of a + :ref:`struct sequence ` object. This information is + accessible as :py:attr:`sys.thread_info` in Python. + + On success, this returns a new :term:`strong reference` to the thread + information; on failure, this returns ``NULL`` with an exception set. + + The caller must hold an :term:`attached thread state`. + + +.. c:macro:: PY_HAVE_THREAD_NATIVE_ID + + This macro is defined when the system supports native thread IDs. + + +.. c:function:: unsigned long PyThread_get_thread_native_id(void) + + Get the native identifier of the current thread as it was assigned by the operating + system's kernel, which will never be less than zero. + + This function is only available when :c:macro:`PY_HAVE_THREAD_NATIVE_ID` is + defined. + + This function cannot fail, and the caller does not need to hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`threading.get_native_id` + + +.. c:function:: void PyThread_exit_thread(void) + + Terminate the current thread. This function is generally considered unsafe + and should be avoided. It is kept solely for backwards compatibility. + + This function is only safe to call if all functions in the full call + stack are written to safely allow it. + + .. warning:: + + If the current system uses POSIX threads (also known as "pthreads"), + this calls :manpage:`pthread_exit(3)`, which attempts to unwind the stack + and call C++ destructors on some libc implementations. However, if a + ``noexcept`` function is reached, it may terminate the process. + Other systems, such as macOS, do unwinding. + + On Windows, this function calls ``_endthreadex()``, which kills the thread + without calling C++ destructors. + + In any case, there is a risk of corruption on the thread's stack. + + .. deprecated:: 3.14 + + +.. c:function:: void PyThread_init_thread(void) + + Initialize ``PyThread*`` APIs. Python executes this function automatically, + so there's little need to call it from an extension module. + + +.. c:function:: int PyThread_set_stacksize(size_t size) + + Set the stack size of the current thread to *size* bytes. + + This function returns ``0`` on success, ``-1`` if *size* is invalid, or + ``-2`` if the system does not support changing the stack size. This function + does not set exceptions. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: size_t PyThread_get_stacksize(void) + + Return the stack size of the current thread in bytes, or ``0`` if the system's + default stack size is in use. + + The caller does not need to hold an :term:`attached thread state`. diff --git a/Doc/c-api/tls.rst b/Doc/c-api/tls.rst new file mode 100644 index 000000000000000..93ac5557141e258 --- /dev/null +++ b/Doc/c-api/tls.rst @@ -0,0 +1,155 @@ +.. highlight:: c + +.. _thread-local-storage: + +Thread-local storage support +============================ + +The Python interpreter provides low-level support for thread-local storage +(TLS) which wraps the underlying native TLS implementation to support the +Python-level thread-local storage API (:class:`threading.local`). The +CPython C level APIs are similar to those offered by pthreads and Windows: +use a thread key and functions to associate a :c:expr:`void*` value per +thread. + +A :term:`thread state` does *not* need to be :term:`attached ` +when calling these functions; they supply their own locking. + +Note that :file:`Python.h` does not include the declaration of the TLS APIs, +you need to include :file:`pythread.h` to use thread-local storage. + +.. note:: + None of these API functions handle memory management on behalf of the + :c:expr:`void*` values. You need to allocate and deallocate them yourself. + If the :c:expr:`void*` values happen to be :c:expr:`PyObject*`, these + functions don't do refcount operations on them either. + +.. _thread-specific-storage-api: + +Thread-specific storage API +--------------------------- + +The thread-specific storage (TSS) API was introduced to supersede the use of the existing TLS API within the +CPython interpreter. This API uses a new type :c:type:`Py_tss_t` instead of +:c:expr:`int` to represent thread keys. + +.. versionadded:: 3.7 + +.. seealso:: "A New C-API for Thread-Local Storage in CPython" (:pep:`539`) + + +.. c:type:: Py_tss_t + + This data structure represents the state of a thread key, the definition of + which may depend on the underlying TLS implementation, and it has an + internal field representing the key's initialization state. There are no + public members in this structure. + + When :ref:`Py_LIMITED_API ` is not defined, static allocation of + this type by :c:macro:`Py_tss_NEEDS_INIT` is allowed. + + +.. c:macro:: Py_tss_NEEDS_INIT + + This macro expands to the initializer for :c:type:`Py_tss_t` variables. + Note that this macro won't be defined with :ref:`Py_LIMITED_API `. + + +Dynamic allocation +------------------ + +Dynamic allocation of the :c:type:`Py_tss_t`, required in extension modules +built with :ref:`Py_LIMITED_API `, where static allocation of this type +is not possible due to its implementation being opaque at build time. + + +.. c:function:: Py_tss_t* PyThread_tss_alloc() + + Return a value which is the same state as a value initialized with + :c:macro:`Py_tss_NEEDS_INIT`, or ``NULL`` in the case of dynamic allocation + failure. + + +.. c:function:: void PyThread_tss_free(Py_tss_t *key) + + Free the given *key* allocated by :c:func:`PyThread_tss_alloc`, after + first calling :c:func:`PyThread_tss_delete` to ensure any associated + thread locals have been unassigned. This is a no-op if the *key* + argument is ``NULL``. + + .. note:: + A freed key becomes a dangling pointer. You should reset the key to + ``NULL``. + + +Methods +------- + +The parameter *key* of these functions must not be ``NULL``. Moreover, the +behaviors of :c:func:`PyThread_tss_set` and :c:func:`PyThread_tss_get` are +undefined if the given :c:type:`Py_tss_t` has not been initialized by +:c:func:`PyThread_tss_create`. + + +.. c:function:: int PyThread_tss_is_created(Py_tss_t *key) + + Return a non-zero value if the given :c:type:`Py_tss_t` has been initialized + by :c:func:`PyThread_tss_create`. + + +.. c:function:: int PyThread_tss_create(Py_tss_t *key) + + Return a zero value on successful initialization of a TSS key. The behavior + is undefined if the value pointed to by the *key* argument is not + initialized by :c:macro:`Py_tss_NEEDS_INIT`. This function can be called + repeatedly on the same key -- calling it on an already initialized key is a + no-op and immediately returns success. + + +.. c:function:: void PyThread_tss_delete(Py_tss_t *key) + + Destroy a TSS key to forget the values associated with the key across all + threads, and change the key's initialization state to uninitialized. A + destroyed key is able to be initialized again by + :c:func:`PyThread_tss_create`. This function can be called repeatedly on + the same key -- calling it on an already destroyed key is a no-op. + + +.. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value) + + Return a zero value to indicate successfully associating a :c:expr:`void*` + value with a TSS key in the current thread. Each thread has a distinct + mapping of the key to a :c:expr:`void*` value. + + +.. c:function:: void* PyThread_tss_get(Py_tss_t *key) + + Return the :c:expr:`void*` value associated with a TSS key in the current + thread. This returns ``NULL`` if no value is associated with the key in the + current thread. + + +.. _thread-local-storage-api: + +Legacy APIs +----------- + +.. deprecated:: 3.7 + This API is superseded by the + :ref:`thread-specific storage (TSS) API `. + +.. note:: + This version of the API does not support platforms where the native TLS key + is defined in a way that cannot be safely cast to ``int``. On such platforms, + :c:func:`PyThread_create_key` will return immediately with a failure status, + and the other TLS functions will all be no-ops on such platforms. + +Due to the compatibility problem noted above, this version of the API should not +be used in new code. + +.. c:function:: int PyThread_create_key() +.. c:function:: void PyThread_delete_key(int key) +.. c:function:: int PyThread_set_key_value(int key, void *value) +.. c:function:: void* PyThread_get_key_value(int key) +.. c:function:: void PyThread_delete_key_value(int key) +.. c:function:: void PyThread_ReInitTLS() diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 815afddad19df1a..ba4c6b93de4c11e 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -37,6 +37,19 @@ Tuple Objects or ``NULL`` with an exception set on failure. +.. c:function:: PyObject* PyTuple_FromArray(PyObject *const *array, Py_ssize_t size) + + Create a tuple of *size* items and copy references from *array* to the new + tuple. + + *array* can be NULL if *size* is ``0``. + + On success, return a new reference. + On error, set an exception and return ``NULL``. + + .. versionadded:: 3.15 + + .. c:function:: PyObject* PyTuple_Pack(Py_ssize_t n, ...) Return a new tuple object of size *n*, @@ -48,7 +61,7 @@ Tuple Objects .. c:function:: Py_ssize_t PyTuple_Size(PyObject *p) Take a pointer to a tuple object, and return the size of that tuple. - On error, return ``-1`` and with an exception set. + On error, return ``-1`` with an exception set. .. c:function:: Py_ssize_t PyTuple_GET_SIZE(PyObject *p) @@ -86,7 +99,8 @@ Tuple Objects Insert a reference to object *o* at position *pos* of the tuple pointed to by *p*. Return ``0`` on success. If *pos* is out of bounds, return ``-1`` - and set an :exc:`IndexError` exception. + and set an :exc:`IndexError` exception. This function should only be used to fill in brand new tuples; + using it on an existing tuple is thread-unsafe. .. note:: @@ -97,7 +111,7 @@ Tuple Objects .. c:function:: void PyTuple_SET_ITEM(PyObject *p, Py_ssize_t pos, PyObject *o) Like :c:func:`PyTuple_SetItem`, but does no error checking, and should *only* be - used to fill in brand new tuples. + used to fill in brand new tuples, using it on an existing tuple is thread-unsafe. Bounds checking is performed as an assertion if Python is built in :ref:`debug mode ` or :option:`with assertions <--with-assertions>`. @@ -135,8 +149,11 @@ Tuple Objects Struct Sequence Objects ----------------------- -Struct sequence objects are the C equivalent of :func:`~collections.namedtuple` -objects, i.e. a sequence whose items can also be accessed through attributes. +A struct sequence object is a :term:`named tuple`, that is, a sequence +whose items can also be accessed through attributes. +It is similar to :func:`collections.namedtuple`, but provides a slightly +different interface. + To create a struct sequence, you first have to create a specific struct sequence type. @@ -220,6 +237,8 @@ type. .. c:function:: PyObject* PyStructSequence_GetItem(PyObject *p, Py_ssize_t pos) Return the object at position *pos* in the struct sequence pointed to by *p*. + The returned reference is borrowed from the struct sequence *p* + (that is: it is only valid as long as you hold a reference to *p*). Bounds checking is performed as an assertion if Python is built in :ref:`debug mode ` or :option:`with assertions <--with-assertions>`. diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 5bdbff4e0ad990d..48eb16bd90834ba 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -3,7 +3,7 @@ .. _typeobjects: Type Objects ------------- +============ .. index:: pair: object; type @@ -110,11 +110,30 @@ Type Objects :c:func:`!_PyType_Lookup` is not called on *type* between the modifications; this is an implementation detail and subject to change.) + The callback is also invoked when a watched heap type is deallocated. + An extension should never call ``PyType_Watch`` with a *watcher_id* that was not returned to it by a previous call to :c:func:`PyType_AddWatcher`. .. versionadded:: 3.12 + .. versionchanged:: 3.15 + The callback is now also invoked when a watched heap type is deallocated. + + +.. c:function:: int PyType_Unwatch(int watcher_id, PyObject *type) + + Mark *type* as not watched. This undoes a previous call to + :c:func:`PyType_Watch`. *type* must not be ``NULL``. + + An extension should never call this function with a *watcher_id* that was + not returned to it by a previous call to :c:func:`PyType_AddWatcher`. + + On success, this function returns ``0``. On failure, this function returns + ``-1`` with an exception set. + + .. versionadded:: 3.12 + .. c:type:: int (*PyType_WatchCallback)(PyObject *type) @@ -124,8 +143,17 @@ Type Objects called on *type* or any type in its MRO; violating this rule could cause infinite recursion. + The callback may be called during type deallocation. In this case, the type + object is temporarily resurrected (its reference count is at least 1) and all + its attributes are still valid. However, the callback should not store new + strong references to the type, as this would resurrect the object and prevent + its deallocation. + .. versionadded:: 3.12 + .. versionchanged:: 3.15 + The callback may now be called during deallocation of a watched heap type. + .. c:function:: int PyType_HasFeature(PyTypeObject *o, int feature) @@ -133,6 +161,18 @@ Type Objects Type features are denoted by single bit flags. +.. c:function:: int PyType_FastSubclass(PyTypeObject *type, int flag) + + Return non-zero if the type object *type* sets the subclass flag *flag*. + Subclass flags are denoted by + :c:macro:`Py_TPFLAGS_*_SUBCLASS `. + This function is used by many ``_Check`` functions for common types. + + .. seealso:: + :c:func:`PyObject_TypeCheck`, which is used as a slower alternative in + ``_Check`` functions for types that don't come with subclass flags. + + .. c:function:: int PyType_IS_GC(PyTypeObject *o) Return true if the type object includes support for the cycle detector; this @@ -169,12 +209,14 @@ Type Objects before initialization) and should be paired with :c:func:`PyObject_Free` in :c:member:`~PyTypeObject.tp_free`. + .. c:function:: PyObject* PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type object. Creates a new instance using the type's :c:member:`~PyTypeObject.tp_alloc` slot and returns the resulting object. + .. c:function:: int PyType_Ready(PyTypeObject *type) Finalize a type object. This should be called on all type objects to finish @@ -191,6 +233,7 @@ Type Objects GC protocol itself by at least implementing the :c:member:`~PyTypeObject.tp_traverse` handle. + .. c:function:: PyObject* PyType_GetName(PyTypeObject *type) Return the type's name. Equivalent to getting the type's @@ -198,6 +241,7 @@ Type Objects .. versionadded:: 3.11 + .. c:function:: PyObject* PyType_GetQualName(PyTypeObject *type) Return the type's qualified name. Equivalent to getting the @@ -213,6 +257,7 @@ Type Objects .. versionadded:: 3.13 + .. c:function:: PyObject* PyType_GetModuleName(PyTypeObject *type) Return the type's module name. Equivalent to getting the @@ -220,6 +265,7 @@ Type Objects .. versionadded:: 3.13 + .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the @@ -236,11 +282,16 @@ Type Objects :c:func:`PyType_GetSlot` can now accept all types. Previously, it was limited to :ref:`heap types `. + .. c:function:: PyObject* PyType_GetModule(PyTypeObject *type) Return the module object associated with the given type when the type was created using :c:func:`PyType_FromModuleAndSpec`. + The returned reference is :term:`borrowed ` from *type*, + and will be valid as long as you hold a reference to *type*. + Do not release it with :c:func:`Py_DECREF` or similar. + If no module is associated with the given type, sets :py:class:`TypeError` and returns ``NULL``. @@ -250,11 +301,12 @@ Type Objects ``Py_TYPE(self)`` may be a *subclass* of the intended class, and subclasses are not necessarily defined in the same module as their superclass. See :c:type:`PyCMethod` to get the class that defines the method. - See :c:func:`PyType_GetModuleByDef` for cases when :c:type:`!PyCMethod` cannot - be used. + See :c:func:`PyType_GetModuleByToken` for cases when :c:type:`!PyCMethod` + cannot be used. .. versionadded:: 3.9 + .. c:function:: void* PyType_GetModuleState(PyTypeObject *type) Return the state of the module object associated with the given type. @@ -269,10 +321,11 @@ Type Objects .. versionadded:: 3.9 -.. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) - Find the first superclass whose module was created from - the given :c:type:`PyModuleDef` *def*, and return that module. +.. c:function:: PyObject* PyType_GetModuleByToken(PyTypeObject *type, const void *mod_token) + + Find the first superclass whose module has the given + :ref:`module token `, and return that module. If no module is found, raises a :py:class:`TypeError` and returns ``NULL``. @@ -282,16 +335,34 @@ Type Objects and other places where a method's defining class cannot be passed using the :c:type:`PyCMethod` calling convention. + .. versionadded:: 3.15 + + +.. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) + + Find the first superclass whose module was created from the given + :c:type:`PyModuleDef` *def*, or whose :ref:`module token ` + is equal to *def*, and return that module. + + Note that modules created from a :c:type:`PyModuleDef` always have their + token set to the :c:type:`PyModuleDef`'s address. + In other words, this function is equivalent to + :c:func:`PyType_GetModuleByToken`, except that it: + + - returns a borrowed reference, and + - has a non-``void*`` argument type (which is a cosmetic difference in C). + The returned reference is :term:`borrowed ` from *type*, and will be valid as long as you hold a reference to *type*. Do not release it with :c:func:`Py_DECREF` or similar. .. versionadded:: 3.11 -.. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) + +.. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *tp_token, PyTypeObject **result) Find the first superclass in *type*'s :term:`method resolution order` whose - :c:macro:`Py_tp_token` token is equal to the given one. + :c:macro:`Py_tp_token` token is equal to *tp_token*. * If found, set *\*result* to a new :term:`strong reference` to it and return ``1``. @@ -302,10 +373,11 @@ Type Objects The *result* argument may be ``NULL``, in which case *\*result* is not set. Use this if you need only the return value. - The *token* argument may not be ``NULL``. + The *tp_token* argument may not be ``NULL``. .. versionadded:: 3.14 + .. c:function:: int PyUnstable_Type_AssignVersionTag(PyTypeObject *type) Attempt to assign a version tag to the given type. @@ -316,36 +388,29 @@ Type Objects .. versionadded:: 3.12 -Creating Heap-Allocated Types -............................. +.. c:function:: int PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) -The following functions and structs are used to create -:ref:`heap types `. + Return true if instances of *type* support creating weak references, false + otherwise. This function always succeeds. *type* must not be ``NULL``. -.. c:function:: PyObject* PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) + .. seealso:: + * :ref:`weakrefobjects` + * :py:mod:`weakref` - Create and return a :ref:`heap type ` from the *spec* - (see :c:macro:`Py_TPFLAGS_HEAPTYPE`). - The metaclass *metaclass* is used to construct the resulting type object. - When *metaclass* is ``NULL``, the metaclass is derived from *bases* - (or *Py_tp_base[s]* slots if *bases* is ``NULL``, see below). +.. _creating-heap-types: - Metaclasses that override :c:member:`~PyTypeObject.tp_new` are not - supported, except if ``tp_new`` is ``NULL``. +Creating Heap-Allocated Types +----------------------------- - The *bases* argument can be used to specify base classes; it can either - be only one class or a tuple of classes. - If *bases* is ``NULL``, the *Py_tp_bases* slot is used instead. - If that also is ``NULL``, the *Py_tp_base* slot is used instead. - If that also is ``NULL``, the new type derives from :class:`object`. +The following function is used to create :ref:`heap types `: - The *module* argument can be used to record the module in which the new - class is defined. It must be a module object or ``NULL``. - If not ``NULL``, the module is associated with the new type and can later be - retrieved with :c:func:`PyType_GetModule`. - The associated module is not inherited by subclasses; it must be specified - for each class individually. +.. c:function:: PyObject *PyType_FromSlots(const PySlot *slots) + + Create and return a :ref:`heap type ` from a :c:type:`!PySlot` + array. + See :ref:`capi-slots` for general information on slots, + and :ref:`pyslot_type_slot_ids` for slots specific to type creation. This function calls :c:func:`PyType_Ready` on the new type. @@ -362,8 +427,408 @@ The following functions and structs are used to create * :py:meth:`~object.__init_subclass__` is not called on any bases. * :py:meth:`~object.__set_name__` is not called on new descriptors. + Slots are typically defined as a global static constant arrays. + However, sometimes slot values are not statically known at compile time. + For example, slots like :c:data:`Py_tp_bases`, :c:data:`Py_tp_metaclass` + and :c:data:`Py_tp_module` require live Python objects. + In this case, it is recommended to put such slots on the stack, + and use :c:macro:`Py_slot_subslots` to refer to an array of static slots. + For example:: + + static const PySlot my_slots[] = { + PySlot_STATIC_DATA(Py_tp_name, "MyClass"), + PySlot_FUNC(Py_tp_repr, my_repr_func), + ... + PySlot_END + }; + + PyObject *make_my_class(PyObject *module) { + PySlot all_slots[] = { + PySlot_STATIC_DATA(Py_slot_subslots, my_slots), + PySlot_DATA(Py_tp_module, module), + PySlot_END + }; + return PyType_FromSlots(all_slots); + } + +Heap types created without the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag may be +modified, for example by setting attributes on them, as with classes defined +in Python code. +Sometimes, such modifications are necessary to fully initialize a type, +but you may wish to prevent users from changing the type after +the initialization is done: + +.. c:function:: int PyType_Freeze(PyTypeObject *type) + + Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. + + All base classes of *type* must be immutable. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + The type must not be used before it's made immutable. For example, type + instances must not be created before the type is made immutable. + + .. versionadded:: 3.14 + + +.. _pyslot_type_slot_ids: + +Type slot IDs +............. + +Most type slot IDs are named like the field names of the structures +:c:type:`PyTypeObject`, :c:type:`PyNumberMethods`, +:c:type:`PySequenceMethods`, :c:type:`PyMappingMethods` and +:c:type:`PyAsyncMethods` with an added ``Py_`` prefix. +For example, use: + +* :c:data:`Py_tp_dealloc` to set :c:member:`PyTypeObject.tp_dealloc` +* :c:data:`Py_nb_add` to set :c:member:`PyNumberMethods.nb_add` +* :c:data:`Py_sq_length` to set :c:member:`PySequenceMethods.sq_length` + +The following slots need additional considerations when specified as slots: + +* :c:data:`Py_tp_name` +* :c:data:`Py_tp_basicsize` and :c:data:`Py_tp_extra_basicsize` +* :c:data:`Py_tp_itemsize` +* :c:data:`Py_tp_flags` + +Additional slots do not directly correspond to a :c:type:`!PyTypeObject` +struct field: + +* :c:data:`Py_tp_token` +* :c:data:`Py_tp_metaclass` +* :c:data:`Py_tp_module` + +The following “offset” fields cannot be set using :c:type:`PyType_Slot`: + +* :c:member:`~PyTypeObject.tp_weaklistoffset` + (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible) +* :c:member:`~PyTypeObject.tp_dictoffset` + (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible) +* :c:member:`~PyTypeObject.tp_vectorcall_offset` + (use ``"__vectorcalloffset__"`` in :ref:`PyMemberDef `) + +If it is not possible to switch to a ``MANAGED`` flag (for example, +for vectorcall or to support Python older than 3.12), specify the +offset in :c:data:`Py_tp_members`. +See :ref:`PyMemberDef documentation ` +for details. + +The following internal fields cannot be set at all when creating a heap +type: + +* :c:member:`~PyTypeObject.tp_dict`, + :c:member:`~PyTypeObject.tp_mro`, + :c:member:`~PyTypeObject.tp_cache`, + :c:member:`~PyTypeObject.tp_subclasses`, and + :c:member:`~PyTypeObject.tp_weaklist`. + +The :c:data:`Py_tp_base` slot is equivalent to :c:data:`Py_tp_bases`; +both may be set either to a type or a tuple of types. +If both are specified, the value of :c:data:`Py_tp_bases` +is used. + +Slot values may not be ``NULL``, except for the following: + +* :c:data:`Py_tp_doc` +* :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` + rather than ``NULL``) + +.. versionchanged:: 3.9 + Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. + +.. versionchanged:: 3.11 + :c:member:`~PyBufferProcs.bf_getbuffer` and + :c:member:`~PyBufferProcs.bf_releasebuffer` are now available + under the :ref:`limited API `. + +.. versionchanged:: 3.14 + The field :c:member:`~PyTypeObject.tp_vectorcall` can now be set + using :c:data:`Py_tp_vectorcall`. See the field's documentation + for details. + +.. versionchanged:: 3.15 + The :c:data:`Py_tp_bases` slot may be set to a single type object, + making it equivalent to the :c:data:`Py_tp_base` slot. + Previously, a tuple of types was required. + +The following slots correspond to fields in the underlying type structure, +but need extra remarks for use as slots: + +.. c:macro:: Py_tp_name + + :c:member:`Slot ID ` for the name of the type, + used to set :c:member:`PyTypeObject.tp_name`. + + This slot (or :c:member:`PyType_Spec.name`) is required to create a type. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:member:`PyType_Spec.name` instead. + + .. impl-detail:: + + CPython processes slots in order. + It is recommended to put ``Py_tp_name`` at the beginning of the slots + array, so that if processing of a later slots fails, error messages + can include the name. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_basicsize + + :c:member:`Slot ID ` for the size of the instance in bytes. + It is used to set :c:member:`PyTypeObject.tp_basicsize`. + + The value must be positive. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:member:`PyType_Spec.basicsize` instead. + + This slot may not be used with :c:func:`PyType_GetSlot`. + Use :c:member:`PyTypeObject.tp_basicsize` instead if needed, but be aware + that a type's size is often considered an implementation detail. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_extra_basicsize + + :c:member:`Slot ID ` for type data size in bytes, that is, + how much space instances of the class need *in addition* + to space needed for superclasses. + + The value is used, together with the size of superclasses, to set + :c:member:`PyTypeObject.tp_basicsize`. + Python will insert padding as needed to meet + :c:member:`!tp_basicsize`'s alignment requirements. + + Use :c:func:`PyObject_GetTypeData` to get a pointer to subclass-specific + memory reserved this way. + + The value must be positive. + To specify that instances need no additional size (that is, size should be + inherited), omit the :c:macro:`!Py_tp_extra_basicsize` slot rather than + set it to zero. + + Specifying both :c:macro:`Py_tp_basicsize` and + :c:macro:`!Py_tp_extra_basicsize` is an error. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use negative :c:member:`PyType_Spec.basicsize` instead. + + This slot may not be used with :c:func:`PyType_GetSlot`. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_itemsize + + :c:member:`Slot ID ` for the size of one element of a + variable-size type, in bytes. + Used to set :c:member:`PyTypeObject.tp_itemsize`. + See :c:member:`!tp_itemsize` documentation for caveats. + + The value must be positive. + + If this slot is missing, :c:member:`~PyTypeObject.tp_itemsize` is inherited. + Extending arbitrary variable-sized classes is dangerous, + since some types use a fixed offset for variable-sized memory, + which can then overlap fixed-sized memory used by a subclass. + To help prevent mistakes, inheriting ``itemsize`` is only possible + in the following situations: + + - The base is not variable-sized (its + :c:member:`~PyTypeObject.tp_itemsize`). + - The requested :c:member:`PyType_Spec.basicsize` is positive, + suggesting that the memory layout of the base class is known. + - The requested :c:member:`PyType_Spec.basicsize` is zero, + suggesting that the subclass does not access the instance's memory + directly. + - With the :c:macro:`Py_TPFLAGS_ITEMS_AT_END` flag. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:member:`PyType_Spec.itemsize` instead. + + This slot may not be used with :c:func:`PyType_GetSlot`. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_flags + + :c:member:`Slot ID ` for type flags, used to set + :c:member:`PyTypeObject.tp_flags`. + + The ``Py_TPFLAGS_HEAPTYPE`` flag is not set, + :c:func:`PyType_FromSpecWithBases` sets it automatically. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use negative :c:member:`PyType_Spec.basicsize` instead. + + This slot may not be used with :c:func:`PyType_GetSlot`. + Use :c:func:`PyType_GetFlags` instead. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_bases + + :c:member:`Slot ID ` for type flags, used to set + :c:member:`PyTypeObject.tp_bases`. + + The slot can be set to a tuple of type objects which the newly created + type should inherit from, like the "positional arguments" of + a Python :ref:`class definition `. + + Alternately, the slot can be set to a single type object to specify + a single base. + The effect is the same as specifying a one-element tuple. + + .. versionchanged:: 3.15 + + Previously, :c:macro:`!Py_tp_bases` required a tuple of types. + +.. c:macro:: Py_tp_base + + Equivalent to :c:macro:`Py_tp_bases` (with ``s`` at the end). + If both are specified, :c:macro:`!Py_tp_bases` takes priority and + this slot is ignored. + + .. versionchanged:: 3.15 + + Previously, :c:macro:`!Py_tp_base` required a single type, not a tuple. + + .. soft-deprecated:: 3.15 + + When not targetting older Python versions, pefer :c:macro:`!Py_tp_bases`. + +The following slots do not correspond to public fields in the +underlying structures: + +.. c:macro:: Py_tp_metaclass + + :c:member:`Slot ID ` for the metaclass used to construct + the resulting type object. + When omitted the metaclass is derived from bases + (:c:macro:`Py_tp_bases` or the *bases* argument of + :c:func:`PyType_FromMetaclass`). + + Metaclasses that override :c:member:`~PyTypeObject.tp_new` are not + supported, except if ``tp_new`` is ``NULL``. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:func:`PyType_FromMetaclass` to specify a metaclass with + :c:type:`!PyType_Spec`. + + This slot may not be used with :c:func:`PyType_GetSlot`. + Use :c:func:`Py_TYPE` on the type object instead. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_module + + :c:member:`Slot ID ` for recording the module in which + the new class is defined. + + The value must be a module object. + The module is associated with the new type and can later be + retrieved with :c:func:`PyType_GetModule`. + The associated module is not inherited by subclasses; it must be specified + for each class individually. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:func:`PyType_FromMetaclass` to specify a module with + :c:type:`!PyType_Spec`. + + This slot may not be used with :c:func:`PyType_GetSlot`. + Use :c:func:`PyType_GetModule` instead. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_token + + :c:member:`Slot ID ` for recording a static memory layout ID + for a class. + + If the class is defined using a :c:type:`PyType_Spec`, and that spec is + statically allocated, the token can be set to the spec using the special + value :c:data:`Py_TP_USE_SPEC`: + + .. code-block:: c + + static PyType_Slot foo_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, + + It can also be set to an arbitrary pointer, but you must ensure that: + + * The pointer outlives the class, so it's not reused for something else + while the class exists. + * It "belongs" to the extension module where the class lives, so it will not + clash with other extensions. + + Use :c:func:`PyType_GetBaseByToken` to check if a class's superclass has + a given token -- that is, check whether the memory layout is compatible. + + To get the token for a given class (without considering superclasses), + use :c:func:`PyType_GetSlot` with ``Py_tp_token``. + + .. versionadded:: 3.14 + + .. c:namespace:: NULL + + .. c:macro:: Py_TP_USE_SPEC + + Used as a value with :c:data:`Py_tp_token` to set the token to the + class's :c:type:`PyType_Spec`. + May only be used for classes defined using :c:type:`!PyType_Spec`. + + Expands to ``NULL``. + + .. versionadded:: 3.14 + +.. c:macro:: Py_tp_slots + + :c:member:`Slot ID ` that works like + :c:macro:`Py_slot_subslots`, except it specifies an array of + :c:type:`PyType_Slot` structures. + + .. versionadded:: 3.15 + + +Soft-deprecated API +------------------- + +The following functions are :term:`soft deprecated`. +They will continue to work, but new features will be added as slots for +:c:func:`PyType_FromSlots`, not as arguments to new ``PyType_From*`` functions. + +.. c:function:: PyObject* PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) + + Create and return a :ref:`heap type ` from the *spec* + (see :c:macro:`Py_TPFLAGS_HEAPTYPE`). + + A non-``NULL`` *metaclass* argument corresponds to the + :c:macro:`Py_tp_metaclass` slot. + + A non-``NULL`` *bases* argument corresponds to the :c:data:`Py_tp_bases` + slot, and takes precedence over :c:data:`Py_tp_bases` and + :c:data:`Py_tp_bases` slots. + + A non-``NULL`` *module* argument corresponds to the + :c:macro:`Py_tp_module` slot. + + This function calls :c:func:`PyType_Ready` on the new type. + + Note that this function does *not* fully match the behavior of + calling :py:class:`type() ` or using the :keyword:`class` statement. + See the note in :c:func:`PyType_FromSlots` documentation for details. + .. versionadded:: 3.12 + .. soft-deprecated:: next + + Prefer :c:func:`PyType_FromSlots` in new code. + + .. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromMetaclass(NULL, module, spec, bases)``. @@ -390,6 +855,11 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. soft-deprecated:: next + + Prefer :c:func:`PyType_FromSlots` in new code. + + .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, bases)``. @@ -411,6 +881,11 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. soft-deprecated:: next + + Prefer :c:func:`PyType_FromSlots` in new code. + + .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, NULL)``. @@ -431,19 +906,9 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. -.. c:function:: int PyType_Freeze(PyTypeObject *type) - - Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. - - All base classes of *type* must be immutable. - - On success, return ``0``. - On error, set an exception and return ``-1``. + .. soft-deprecated:: next - The type must not be used before it's made immutable. For example, type - instances must not be created before the type is made immutable. - - .. versionadded:: 3.14 + Prefer :c:func:`PyType_FromSlots` in new code. .. raw:: html @@ -456,27 +921,23 @@ The following functions and structs are used to create .. c:type:: PyType_Spec - Structure defining a type's behavior. + Structure defining a type's behavior, used for soft-deprecated functions + like :c:func:`PyType_FromMetaclass`. + + This structure contains several members that can instead be specified + as :ref:`slots ` for :c:func:`PyType_FromSlots`, + and an array of slot entries with a simpler structure. .. c:member:: const char* name - Name of the type, used to set :c:member:`PyTypeObject.tp_name`. + Corresponds to :c:macro:`Py_tp_name`. .. c:member:: int basicsize - If positive, specifies the size of the instance in bytes. - It is used to set :c:member:`PyTypeObject.tp_basicsize`. - - If zero, specifies that :c:member:`~PyTypeObject.tp_basicsize` - should be inherited. + If positive, corresponds to :c:macro:`Py_tp_basicsize`. - If negative, the absolute value specifies how much space instances of the - class need *in addition* to the superclass. - Use :c:func:`PyObject_GetTypeData` to get a pointer to subclass-specific - memory reserved this way. - For negative :c:member:`!basicsize`, Python will insert padding when - needed to meet :c:member:`~PyTypeObject.tp_basicsize`'s alignment - requirements. + If negative, corresponds to :c:macro:`Py_tp_extra_basicsize` set to + the absolute value. .. versionchanged:: 3.12 @@ -484,159 +945,53 @@ The following functions and structs are used to create .. c:member:: int itemsize - Size of one element of a variable-size type, in bytes. - Used to set :c:member:`PyTypeObject.tp_itemsize`. - See ``tp_itemsize`` documentation for caveats. - - If zero, :c:member:`~PyTypeObject.tp_itemsize` is inherited. - Extending arbitrary variable-sized classes is dangerous, - since some types use a fixed offset for variable-sized memory, - which can then overlap fixed-sized memory used by a subclass. - To help prevent mistakes, inheriting ``itemsize`` is only possible - in the following situations: - - - The base is not variable-sized (its - :c:member:`~PyTypeObject.tp_itemsize`). - - The requested :c:member:`PyType_Spec.basicsize` is positive, - suggesting that the memory layout of the base class is known. - - The requested :c:member:`PyType_Spec.basicsize` is zero, - suggesting that the subclass does not access the instance's memory - directly. - - With the :c:macro:`Py_TPFLAGS_ITEMS_AT_END` flag. + Corresponds to :c:macro:`Py_tp_itemsize`. .. c:member:: unsigned int flags - Type flags, used to set :c:member:`PyTypeObject.tp_flags`. - - If the ``Py_TPFLAGS_HEAPTYPE`` flag is not set, - :c:func:`PyType_FromSpecWithBases` sets it automatically. + Corresponds to :c:macro:`Py_tp_flags`. .. c:member:: PyType_Slot *slots - Array of :c:type:`PyType_Slot` structures. - Terminated by the special slot value ``{0, NULL}``. + Array of :c:type:`PyType_Slot` (not :c:type:`PySlot`) structures. + Terminated by the special slot value ``{0, NULL}``. Each slot ID should be specified at most once. -.. raw:: html - - - - - -.. c:type:: PyType_Slot - - Structure defining optional functionality of a type, containing a slot ID - and a value pointer. - - .. c:member:: int slot - - A slot ID. - - Slot IDs are named like the field names of the structures - :c:type:`PyTypeObject`, :c:type:`PyNumberMethods`, - :c:type:`PySequenceMethods`, :c:type:`PyMappingMethods` and - :c:type:`PyAsyncMethods` with an added ``Py_`` prefix. - For example, use: - - * ``Py_tp_dealloc`` to set :c:member:`PyTypeObject.tp_dealloc` - * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` - * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` - - An additional slot is supported that does not correspond to a - :c:type:`!PyTypeObject` struct field: - - * :c:data:`Py_tp_token` - - The following “offset” fields cannot be set using :c:type:`PyType_Slot`: + .. c:namespace:: NULL - * :c:member:`~PyTypeObject.tp_weaklistoffset` - (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible) - * :c:member:`~PyTypeObject.tp_dictoffset` - (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible) - * :c:member:`~PyTypeObject.tp_vectorcall_offset` - (use ``"__vectorcalloffset__"`` in - :ref:`PyMemberDef `) + .. raw:: html - If it is not possible to switch to a ``MANAGED`` flag (for example, - for vectorcall or to support Python older than 3.12), specify the - offset in :c:member:`Py_tp_members `. - See :ref:`PyMemberDef documentation ` - for details. + + + - The following internal fields cannot be set at all when creating a heap - type: + .. c:type:: PyType_Slot - * :c:member:`~PyTypeObject.tp_dict`, - :c:member:`~PyTypeObject.tp_mro`, - :c:member:`~PyTypeObject.tp_cache`, - :c:member:`~PyTypeObject.tp_subclasses`, and - :c:member:`~PyTypeObject.tp_weaklist`. + Structure defining optional functionality of a type, used for + soft-deprecated functions like :c:func:`PyType_FromMetaclass`. - Setting :c:data:`Py_tp_bases` or :c:data:`Py_tp_base` may be - problematic on some platforms. - To avoid issues, use the *bases* argument of - :c:func:`PyType_FromSpecWithBases` instead. + Note that a :c:type:`!PyType_Slot` array may be included in a + :c:type:`!PySlot` array using :c:macro:`Py_tp_slots`, + and vice versa using :c:macro:`Py_slot_subslots`. - .. versionchanged:: 3.9 - Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. + Each :c:type:`!PyType_Slot` structure ``tpslot`` is interpreted + as the following :c:type:`PySlot` structure:: - .. versionchanged:: 3.11 - :c:member:`~PyBufferProcs.bf_getbuffer` and - :c:member:`~PyBufferProcs.bf_releasebuffer` are now available - under the :ref:`limited API `. + (PySlot){ + .sl_id=tpslot.slot, + .sl_flags=PySlot_INTPTR | sub_static, + .sl_ptr=tpslot.func + } - .. versionchanged:: 3.14 - The field :c:member:`~PyTypeObject.tp_vectorcall` can now set - using ``Py_tp_vectorcall``. See the field's documentation - for details. + where ``sub_static`` is ``PySlot_STATIC`` if the slot requires + the flag (such as for :c:macro:`Py_tp_methods`), or if this flag + is present on the "parent" :c:macro:`!Py_tp_slots` slot (if any). - .. c:member:: void *pfunc + .. c:member:: int slot - The desired value of the slot. In most cases, this is a pointer - to a function. + Corresponds to :c:member:`PySlot.sl_id`. - *pfunc* values may not be ``NULL``, except for the following slots: + .. c:member:: void *pfunc - * ``Py_tp_doc`` - * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` - rather than ``NULL``) - -.. c:macro:: Py_tp_token - - A :c:member:`~PyType_Slot.slot` that records a static memory layout ID - for a class. - - If the :c:type:`PyType_Spec` of the class is statically - allocated, the token can be set to the spec using the special value - :c:data:`Py_TP_USE_SPEC`: - - .. code-block:: c - - static PyType_Slot foo_slots[] = { - {Py_tp_token, Py_TP_USE_SPEC}, - - It can also be set to an arbitrary pointer, but you must ensure that: - - * The pointer outlives the class, so it's not reused for something else - while the class exists. - * It "belongs" to the extension module where the class lives, so it will not - clash with other extensions. - - Use :c:func:`PyType_GetBaseByToken` to check if a class's superclass has - a given token -- that is, check whether the memory layout is compatible. - - To get the token for a given class (without considering superclasses), - use :c:func:`PyType_GetSlot` with ``Py_tp_token``. - - .. versionadded:: 3.14 - - .. c:namespace:: NULL - - .. c:macro:: Py_TP_USE_SPEC - - Used as a value with :c:data:`Py_tp_token` to set the token to the - class's :c:type:`PyType_Spec`. - Expands to ``NULL``. - - .. versionadded:: 3.14 + Corresponds to :c:member:`PySlot.sl_ptr`. diff --git a/Doc/c-api/typehints.rst b/Doc/c-api/typehints.rst index 98fe68737deb81c..ec2fba6da8b0438 100644 --- a/Doc/c-api/typehints.rst +++ b/Doc/c-api/typehints.rst @@ -31,7 +31,7 @@ two types exist -- :ref:`GenericAlias ` and static PyMethodDef my_obj_methods[] = { // Other methods. ... - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, "See PEP 585"} + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, "my_obj is generic over its contained type"} ... } diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 060d6f60174b41a..16dcb880712d244 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -555,6 +555,9 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: const char* PyTypeObject.tp_name + See :c:macro:`Py_tp_name` for the corresponding + :c:member:`Slot ID `. + Pointer to a NUL-terminated string containing the name of the type. For types that are accessible as module globals, the string should be the full module name, followed by a dot, followed by the type name; for built-in types, it @@ -594,6 +597,10 @@ and :c:data:`PyType_Type` effectively act as defaults.) These fields allow calculating the size in bytes of instances of the type. + See :c:macro:`Py_tp_basicsize`, :c:macro:`Py_tp_extra_basicsize` and + :c:macro:`Py_tp_itemsize` for the corresponding + :c:member:`Slot IDs `. + There are two kinds of types: types with fixed-length instances have a zero :c:member:`!tp_itemsize` field, types with variable-length instances have a non-zero :c:member:`!tp_itemsize` field. For a type with fixed-length instances, all @@ -676,6 +683,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_dealloc + .. corresponding-type-slot:: Py_tp_dealloc + A pointer to the instance destructor function. The function signature is:: void tp_dealloc(PyObject *self); @@ -860,6 +869,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getattrfunc PyTypeObject.tp_getattr + .. corresponding-type-slot:: Py_tp_getattr + An optional pointer to the get-attribute-string function. This field is deprecated. When it is defined, it should point to a function @@ -877,6 +888,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: setattrfunc PyTypeObject.tp_setattr + .. corresponding-type-slot:: Py_tp_setattr + An optional pointer to the function for setting and deleting attributes. This field is deprecated. When it is defined, it should point to a function @@ -909,6 +922,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: reprfunc PyTypeObject.tp_repr + .. corresponding-type-slot:: Py_tp_repr + .. index:: pair: built-in function; repr An optional pointer to a function that implements the built-in function @@ -974,6 +989,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: hashfunc PyTypeObject.tp_hash + .. corresponding-type-slot:: Py_tp_hash + .. index:: pair: built-in function; hash An optional pointer to a function that implements the built-in function @@ -1015,6 +1032,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: ternaryfunc PyTypeObject.tp_call + .. corresponding-type-slot:: Py_tp_call + An optional pointer to a function that implements calling the object. This should be ``NULL`` if the object is not callable. The signature is the same as for :c:func:`PyObject_Call`:: @@ -1028,6 +1047,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: reprfunc PyTypeObject.tp_str + .. corresponding-type-slot:: Py_tp_str + An optional pointer to a function that implements the built-in operation :func:`str`. (Note that :class:`str` is a type now, and :func:`str` calls the constructor for that type. This constructor calls :c:func:`PyObject_Str` to do @@ -1053,6 +1074,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getattrofunc PyTypeObject.tp_getattro + .. corresponding-type-slot:: Py_tp_getattro + An optional pointer to the get-attribute function. The signature is the same as for :c:func:`PyObject_GetAttr`:: @@ -1077,6 +1100,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: setattrofunc PyTypeObject.tp_setattro + .. corresponding-type-slot:: Py_tp_setattro + An optional pointer to the function for setting and deleting attributes. The signature is the same as for :c:func:`PyObject_SetAttr`:: @@ -1115,6 +1140,9 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: unsigned long PyTypeObject.tp_flags + See :c:macro:`Py_tp_flags` for the corresponding + :c:member:`Slot ID `. + This field is a bit mask of various flags. Some flags indicate variant semantics for certain situations; others are used to indicate that certain fields in the type object (or in the extension structures referenced via @@ -1260,7 +1288,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) This bit indicates that instances of the class have a :attr:`~object.__dict__` attribute, and that the space for the dictionary is managed by the VM. - If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` should also be set. + If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` must also be set. The type traverse function must call :c:func:`PyObject_VisitManagedDict` and its clear function must call :c:func:`PyObject_ClearManagedDict`. @@ -1278,6 +1306,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) This bit indicates that instances of the class should be weakly referenceable. + If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` must also be set. + .. versionadded:: 3.12 **Inheritance:** @@ -1286,6 +1316,19 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_weaklistoffset` field is set in a superclass. + .. c:macro:: Py_TPFLAGS_PREHEADER + + These bits indicate that the VM will manage some fields by storing them + before the object. Currently, this macro is equivalent to + :c:expr:`Py_TPFLAGS_MANAGED_DICT | Py_TPFLAGS_MANAGED_WEAKREF`. + + This macro value relies on the implementation of the VM, so its value is not + stable and may change in a future version. Prefer using individual + flags instead. + + .. versionadded:: 3.12 + + .. c:macro:: Py_TPFLAGS_ITEMS_AT_END Only usable with variable-size types, i.e. ones with non-zero @@ -1318,8 +1361,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_BASE_EXC_SUBCLASS .. c:macro:: Py_TPFLAGS_TYPE_SUBCLASS - These flags are used by functions such as - :c:func:`PyLong_Check` to quickly determine if a type is a subclass + Functions such as :c:func:`PyLong_Check` will call :c:func:`PyType_FastSubclass` + with one of these flags to quickly determine if a type is a subclass of a built-in type; such specific checks are faster than a generic check, like :c:func:`PyObject_IsInstance`. Custom types that inherit from built-ins should have their :c:member:`~PyTypeObject.tp_flags` @@ -1340,6 +1383,9 @@ and :c:data:`PyType_Type` effectively act as defaults.) type structure. + .. c:macro:: _Py_TPFLAGS_HAVE_VECTORCALL + :no-typesetting: + .. c:macro:: Py_TPFLAGS_HAVE_VECTORCALL This bit is set when the class implements @@ -1351,7 +1397,12 @@ and :c:data:`PyType_Type` effectively act as defaults.) This bit is inherited if :c:member:`~PyTypeObject.tp_call` is also inherited. - .. versionadded:: 3.9 + .. versionadded:: 3.8 as ``_Py_TPFLAGS_HAVE_VECTORCALL`` + + .. versionchanged:: 3.9 + + Renamed to the current name, without the leading underscore. + The old provisional name is :term:`soft deprecated`. .. versionchanged:: 3.12 @@ -1458,102 +1509,75 @@ and :c:data:`PyType_Type` effectively act as defaults.) It will be removed in a future version of CPython -.. c:member:: const char* PyTypeObject.tp_doc + .. c:macro:: Py_TPFLAGS_HAVE_VERSION_TAG - An optional pointer to a NUL-terminated C string giving the docstring for this - type object. This is exposed as the :attr:`~type.__doc__` attribute on the - type and instances of the type. + This macro does nothing. + Historically, this would indicate that the + :c:member:`~PyTypeObject.tp_version_tag` field was available and + initialized. - **Inheritance:** + .. soft-deprecated:: 3.13 - This field is *not* inherited by subtypes. + .. c:macro:: Py_TPFLAGS_INLINE_VALUES -.. c:member:: traverseproc PyTypeObject.tp_traverse + This bit indicates that instances of this type will have an "inline values" + array (containing the object's attributes) placed directly after the end + of the object. - An optional pointer to a traversal function for the garbage collector. This is - only used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: + This requires that :c:macro:`Py_TPFLAGS_HAVE_GC` is set. - int tp_traverse(PyObject *self, visitproc visit, void *arg); + **Inheritance:** - More information about Python's garbage collection scheme can be found - in section :ref:`supporting-cycle-detection`. + This flag is not inherited. - The :c:member:`~PyTypeObject.tp_traverse` pointer is used by the garbage collector to detect - reference cycles. A typical implementation of a :c:member:`~PyTypeObject.tp_traverse` function - simply calls :c:func:`Py_VISIT` on each of the instance's members that are Python - objects that the instance owns. For example, this is function :c:func:`!local_traverse` from the - :mod:`!_thread` extension module:: + .. versionadded:: 3.13 - static int - local_traverse(PyObject *op, visitproc visit, void *arg) - { - localobject *self = (localobject *) op; - Py_VISIT(self->args); - Py_VISIT(self->kw); - Py_VISIT(self->dict); - return 0; - } - Note that :c:func:`Py_VISIT` is called only on those members that can participate - in reference cycles. Although there is also a ``self->key`` member, it can only - be ``NULL`` or a Python string and therefore cannot be part of a reference cycle. + .. c:macro:: Py_TPFLAGS_IS_ABSTRACT - On the other hand, even if you know a member can never be part of a cycle, as a - debugging aid you may want to visit it anyway just so the :mod:`gc` module's - :func:`~gc.get_referents` function will include it. + This bit indicates that this is an abstract type and therefore cannot + be instantiated. - Heap types (:c:macro:`Py_TPFLAGS_HEAPTYPE`) must visit their type with:: + **Inheritance:** - Py_VISIT(Py_TYPE(self)); + This flag is not inherited. - It is only needed since Python 3.9. To support Python 3.8 and older, this - line must be conditional:: + .. seealso:: + :mod:`abc` - #if PY_VERSION_HEX >= 0x03090000 - Py_VISIT(Py_TYPE(self)); - #endif - If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the - :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call - :c:func:`PyObject_VisitManagedDict` like this:: + .. c:macro:: Py_TPFLAGS_HAVE_STACKLESS_EXTENSION - PyObject_VisitManagedDict((PyObject*)self, visit, arg); + Internal. Do not set or unset this flag. + Historically, this was a reserved flag for use in Stackless Python. - .. warning:: - When implementing :c:member:`~PyTypeObject.tp_traverse`, only the - members that the instance *owns* (by having :term:`strong references - ` to them) must be - visited. For instance, if an object supports weak references via the - :c:member:`~PyTypeObject.tp_weaklist` slot, the pointer supporting - the linked list (what *tp_weaklist* points to) must **not** be - visited as the instance does not directly own the weak references to itself - (the weakreference list is there to support the weak reference machinery, - but the instance has no strong reference to the elements inside it, as they - are allowed to be removed even if the instance is still alive). - - Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to - :c:func:`!local_traverse` to have these specific names; don't name them just - anything. - - Instances of :ref:`heap-allocated types ` hold a reference to - their type. Their traversal function must therefore either visit - :c:func:`Py_TYPE(self) `, or delegate this responsibility by - calling ``tp_traverse`` of another heap-allocated type (such as a - heap-allocated superclass). - If they do not, the type object may not be garbage-collected. + .. warning:: + This flag is present in header files, but is not be used. + This may be removed in a future version of CPython. - .. note:: - The :c:member:`~PyTypeObject.tp_traverse` function can be called from any - thread. +.. c:member:: const char* PyTypeObject.tp_doc - .. versionchanged:: 3.9 + .. corresponding-type-slot:: Py_tp_doc - Heap-allocated types are expected to visit ``Py_TYPE(self)`` in - ``tp_traverse``. In earlier versions of Python, due to - `bug 40217 `_, doing this - may lead to crashes in subclasses. + An optional pointer to a NUL-terminated C string giving the docstring for this + type object. This is exposed as the :attr:`~type.__doc__` attribute on the + type and instances of the type. + + **Inheritance:** + + This field is *not* inherited by subtypes. + + +.. c:member:: traverseproc PyTypeObject.tp_traverse + + .. corresponding-type-slot:: Py_tp_traverse + + An optional pointer to a traversal function for the garbage collector. This is + only used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. + + See :ref:`gc-traversal` for documentation. **Inheritance:** @@ -1567,6 +1591,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: inquiry PyTypeObject.tp_clear + .. corresponding-type-slot:: Py_tp_clear + An optional pointer to a clear function. The signature is:: int tp_clear(PyObject *); @@ -1691,7 +1717,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:func:`Py_CLEAR` macro performs the operations in a safe order. If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the - :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call + :c:member:`~PyTypeObject.tp_flags` field, the clear function must call :c:func:`PyObject_ClearManagedDict` like this:: PyObject_ClearManagedDict((PyObject*)self); @@ -1715,6 +1741,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: richcmpfunc PyTypeObject.tp_richcompare + .. corresponding-type-slot:: Py_tp_richcompare + An optional pointer to the rich comparison function, whose signature is:: PyObject *tp_richcompare(PyObject *self, PyObject *other, int op); @@ -1817,6 +1845,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getiterfunc PyTypeObject.tp_iter + .. corresponding-type-slot:: Py_tp_iter + An optional pointer to a function that returns an :term:`iterator` for the object. Its presence normally signals that the instances of this type are :term:`iterable` (although sequences may be iterable without this function). @@ -1832,6 +1862,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: iternextfunc PyTypeObject.tp_iternext + .. corresponding-type-slot:: Py_tp_iternext + An optional pointer to a function that returns the next item in an :term:`iterator`. The signature is:: @@ -1855,6 +1887,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyMethodDef* PyTypeObject.tp_methods + .. corresponding-type-slot:: Py_tp_methods + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMethodDef` structures, declaring regular methods of this type. @@ -1869,6 +1903,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyMemberDef* PyTypeObject.tp_members + .. corresponding-type-slot:: Py_tp_members + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMemberDef` structures, declaring regular data members (fields or slots) of instances of this type. @@ -1884,6 +1920,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyGetSetDef* PyTypeObject.tp_getset + .. corresponding-type-slot:: Py_tp_getset + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyGetSetDef` structures, declaring computed attributes of instances of this type. @@ -1902,6 +1940,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) this level, only single inheritance is supported; multiple inheritance require dynamically creating a type object by calling the metatype. + For the corresponding slot ID, see :c:macro:`Py_tp_base`. + .. note:: .. from Modules/xxmodule.c @@ -1970,6 +2010,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: descrgetfunc PyTypeObject.tp_descr_get + .. corresponding-type-slot:: Py_tp_descr_get + An optional pointer to a "descriptor get" function. The function signature is:: @@ -1985,6 +2027,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: descrsetfunc PyTypeObject.tp_descr_set + .. corresponding-type-slot:: Py_tp_descr_set + An optional pointer to a function for setting and deleting a descriptor's value. @@ -2045,6 +2089,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: initproc PyTypeObject.tp_init + .. corresponding-type-slot:: Py_tp_init + An optional pointer to an instance initialization function. This function corresponds to the :meth:`~object.__init__` method of classes. Like @@ -2080,6 +2126,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: allocfunc PyTypeObject.tp_alloc + .. corresponding-type-slot:: Py_tp_alloc + An optional pointer to an instance allocation function. The function signature is:: @@ -2103,6 +2151,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: newfunc PyTypeObject.tp_new + .. corresponding-type-slot:: Py_tp_new + An optional pointer to an instance creation function. The function signature is:: @@ -2142,6 +2192,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: freefunc PyTypeObject.tp_free + .. corresponding-type-slot:: Py_tp_free + An optional pointer to an instance deallocation function. Its signature is:: void tp_free(void *self); @@ -2171,6 +2223,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: inquiry PyTypeObject.tp_is_gc + .. corresponding-type-slot:: Py_tp_is_gc + An optional pointer to a function called by the garbage collector. The garbage collector needs to know whether a particular object is collectible @@ -2204,10 +2258,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) This field should be set to ``NULL`` and treated as read-only. Python will fill it in when the type is :c:func:`initialized `. - For dynamically created classes, the ``Py_tp_bases`` - :c:type:`slot ` can be used instead of the *bases* argument - of :c:func:`PyType_FromSpecWithBases`. - The argument form is preferred. + For the corresponding slot ID, see :c:macro:`Py_tp_bases`. .. warning:: @@ -2279,6 +2330,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_del + .. corresponding-type-slot:: Py_tp_del + This field is deprecated. Use :c:member:`~PyTypeObject.tp_finalize` instead. @@ -2293,6 +2346,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_finalize + .. corresponding-type-slot:: Py_tp_finalize + An optional pointer to an instance finalization function. This is the C implementation of the :meth:`~object.__del__` special method. Its signature is:: @@ -2451,6 +2506,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: vectorcallfunc PyTypeObject.tp_vectorcall + .. corresponding-type-slot:: Py_tp_vectorcall + A :ref:`vectorcall function ` to use for calls of this type object (rather than instances). In other words, ``tp_vectorcall`` can be used to optimize ``type.__call__``, @@ -2544,9 +2601,6 @@ This is done by filling a :c:type:`PyType_Spec` structure and calling Number Object Structures ------------------------ -.. sectionauthor:: Amaury Forgeot d'Arc - - .. c:type:: PyNumberMethods This structure holds pointers to the functions which an object uses to @@ -2616,51 +2670,154 @@ Number Object Structures Python 3.0.1. .. c:member:: binaryfunc PyNumberMethods.nb_add + + .. corresponding-type-slot:: Py_nb_add + .. c:member:: binaryfunc PyNumberMethods.nb_subtract + + .. corresponding-type-slot:: Py_nb_subtract + .. c:member:: binaryfunc PyNumberMethods.nb_multiply + + .. corresponding-type-slot:: Py_nb_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_remainder + + .. corresponding-type-slot:: Py_nb_remainder + .. c:member:: binaryfunc PyNumberMethods.nb_divmod + + .. corresponding-type-slot:: Py_nb_divmod + .. c:member:: ternaryfunc PyNumberMethods.nb_power + + .. corresponding-type-slot:: Py_nb_power + .. c:member:: unaryfunc PyNumberMethods.nb_negative + + .. corresponding-type-slot:: Py_nb_negative + .. c:member:: unaryfunc PyNumberMethods.nb_positive + + .. corresponding-type-slot:: Py_nb_positive + .. c:member:: unaryfunc PyNumberMethods.nb_absolute + + .. corresponding-type-slot:: Py_nb_absolute + .. c:member:: inquiry PyNumberMethods.nb_bool + + .. corresponding-type-slot:: Py_nb_bool + .. c:member:: unaryfunc PyNumberMethods.nb_invert + + .. corresponding-type-slot:: Py_nb_invert + .. c:member:: binaryfunc PyNumberMethods.nb_lshift + + .. corresponding-type-slot:: Py_nb_lshift + .. c:member:: binaryfunc PyNumberMethods.nb_rshift + + .. corresponding-type-slot:: Py_nb_rshift + .. c:member:: binaryfunc PyNumberMethods.nb_and + + .. corresponding-type-slot:: Py_nb_and + .. c:member:: binaryfunc PyNumberMethods.nb_xor + + .. corresponding-type-slot:: Py_nb_xor + .. c:member:: binaryfunc PyNumberMethods.nb_or + + .. corresponding-type-slot:: Py_nb_or + .. c:member:: unaryfunc PyNumberMethods.nb_int + + .. corresponding-type-slot:: Py_nb_int + .. c:member:: void *PyNumberMethods.nb_reserved + .. c:member:: unaryfunc PyNumberMethods.nb_float + + .. corresponding-type-slot:: Py_nb_float + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_add + + .. corresponding-type-slot:: Py_nb_inplace_add + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_subtract + + .. corresponding-type-slot:: Py_nb_inplace_subtract + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_multiply + + .. corresponding-type-slot:: Py_nb_inplace_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_remainder + + .. corresponding-type-slot:: Py_nb_inplace_remainder + .. c:member:: ternaryfunc PyNumberMethods.nb_inplace_power + + .. corresponding-type-slot:: Py_nb_inplace_power + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_lshift + + .. corresponding-type-slot:: Py_nb_inplace_lshift + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_rshift + + .. corresponding-type-slot:: Py_nb_inplace_rshift + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_and + + .. corresponding-type-slot:: Py_nb_inplace_and + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_xor + + .. corresponding-type-slot:: Py_nb_inplace_xor + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_or + + .. corresponding-type-slot:: Py_nb_inplace_or + .. c:member:: binaryfunc PyNumberMethods.nb_floor_divide + + .. corresponding-type-slot:: Py_nb_floor_divide + .. c:member:: binaryfunc PyNumberMethods.nb_true_divide + + .. corresponding-type-slot:: Py_nb_true_divide + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_floor_divide + + .. corresponding-type-slot:: Py_nb_inplace_floor_divide + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_true_divide + + .. corresponding-type-slot:: Py_nb_inplace_true_divide + .. c:member:: unaryfunc PyNumberMethods.nb_index + + .. corresponding-type-slot:: Py_nb_index + .. c:member:: binaryfunc PyNumberMethods.nb_matrix_multiply + + .. corresponding-type-slot:: Py_nb_matrix_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_matrix_multiply + .. corresponding-type-slot:: Py_nb_inplace_matrix_multiply + + .. _mapping-structs: Mapping Object Structures ------------------------- -.. sectionauthor:: Amaury Forgeot d'Arc - - .. c:type:: PyMappingMethods This structure holds pointers to the functions which an object uses to @@ -2668,12 +2825,16 @@ Mapping Object Structures .. c:member:: lenfunc PyMappingMethods.mp_length + .. corresponding-type-slot:: Py_mp_length + This function is used by :c:func:`PyMapping_Size` and :c:func:`PyObject_Size`, and has the same signature. This slot may be set to ``NULL`` if the object has no defined length. .. c:member:: binaryfunc PyMappingMethods.mp_subscript + .. corresponding-type-slot:: Py_mp_subscript + This function is used by :c:func:`PyObject_GetItem` and :c:func:`PySequence_GetSlice`, and has the same signature as :c:func:`!PyObject_GetItem`. This slot must be filled for the @@ -2682,6 +2843,8 @@ Mapping Object Structures .. c:member:: objobjargproc PyMappingMethods.mp_ass_subscript + .. corresponding-type-slot:: Py_mp_ass_subscript + This function is used by :c:func:`PyObject_SetItem`, :c:func:`PyObject_DelItem`, :c:func:`PySequence_SetSlice` and :c:func:`PySequence_DelSlice`. It has the same signature as @@ -2695,9 +2858,6 @@ Mapping Object Structures Sequence Object Structures -------------------------- -.. sectionauthor:: Amaury Forgeot d'Arc - - .. c:type:: PySequenceMethods This structure holds pointers to the functions which an object uses to @@ -2705,6 +2865,8 @@ Sequence Object Structures .. c:member:: lenfunc PySequenceMethods.sq_length + .. corresponding-type-slot:: Py_sq_length + This function is used by :c:func:`PySequence_Size` and :c:func:`PyObject_Size`, and has the same signature. It is also used for handling negative indices via the :c:member:`~PySequenceMethods.sq_item` @@ -2712,18 +2874,24 @@ Sequence Object Structures .. c:member:: binaryfunc PySequenceMethods.sq_concat + .. corresponding-type-slot:: Py_sq_concat + This function is used by :c:func:`PySequence_Concat` and has the same signature. It is also used by the ``+`` operator, after trying the numeric addition via the :c:member:`~PyNumberMethods.nb_add` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_repeat + .. corresponding-type-slot:: Py_sq_repeat + This function is used by :c:func:`PySequence_Repeat` and has the same signature. It is also used by the ``*`` operator, after trying numeric multiplication via the :c:member:`~PyNumberMethods.nb_multiply` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_item + .. corresponding-type-slot:: Py_sq_item + This function is used by :c:func:`PySequence_GetItem` and has the same signature. It is also used by :c:func:`PyObject_GetItem`, after trying the subscription via the :c:member:`~PyMappingMethods.mp_subscript` slot. @@ -2737,6 +2905,8 @@ Sequence Object Structures .. c:member:: ssizeobjargproc PySequenceMethods.sq_ass_item + .. corresponding-type-slot:: Py_sq_ass_item + This function is used by :c:func:`PySequence_SetItem` and has the same signature. It is also used by :c:func:`PyObject_SetItem` and :c:func:`PyObject_DelItem`, after trying the item assignment and deletion @@ -2746,6 +2916,8 @@ Sequence Object Structures .. c:member:: objobjproc PySequenceMethods.sq_contains + .. corresponding-type-slot:: Py_sq_contains + This function may be used by :c:func:`PySequence_Contains` and has the same signature. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_Contains` simply traverses the sequence until it @@ -2753,6 +2925,8 @@ Sequence Object Structures .. c:member:: binaryfunc PySequenceMethods.sq_inplace_concat + .. corresponding-type-slot:: Py_sq_inplace_concat + This function is used by :c:func:`PySequence_InPlaceConcat` and has the same signature. It should modify its first operand, and return it. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceConcat` @@ -2762,6 +2936,8 @@ Sequence Object Structures .. c:member:: ssizeargfunc PySequenceMethods.sq_inplace_repeat + .. corresponding-type-slot:: Py_sq_inplace_repeat + This function is used by :c:func:`PySequence_InPlaceRepeat` and has the same signature. It should modify its first operand, and return it. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceRepeat` @@ -2775,10 +2951,6 @@ Sequence Object Structures Buffer Object Structures ------------------------ -.. sectionauthor:: Greg J. Stein -.. sectionauthor:: Benjamin Peterson -.. sectionauthor:: Stefan Krah - .. c:type:: PyBufferProcs This structure holds pointers to the functions required by the @@ -2787,6 +2959,8 @@ Buffer Object Structures .. c:member:: getbufferproc PyBufferProcs.bf_getbuffer + .. corresponding-type-slot:: Py_bf_getbuffer + The signature of this function is:: int (PyObject *exporter, Py_buffer *view, int flags); @@ -2796,24 +2970,42 @@ Buffer Object Structures steps: (1) Check if the request can be met. If not, raise :exc:`BufferError`, - set :c:expr:`view->obj` to ``NULL`` and return ``-1``. + set ``view->obj`` to ``NULL`` and return ``-1``. (2) Fill in the requested fields. (3) Increment an internal counter for the number of exports. - (4) Set :c:expr:`view->obj` to *exporter* and increment :c:expr:`view->obj`. + (4) Set ``view->obj`` to *exporter* and increment ``view->obj``. (5) Return ``0``. + **Thread safety:** + + In the :term:`free-threaded build`, implementations must ensure: + + * The export counter increment in step (3) is atomic. + + * The underlying buffer data remains valid and at a stable memory + location for the lifetime of all exports. + + * For objects that support resizing or reallocation (such as + :class:`bytearray`), the export counter is checked atomically before + such operations, and :exc:`BufferError` is raised if exports exist. + + * The function is safe to call concurrently from multiple threads. + + See also :ref:`thread-safety-memoryview` for the Python-level + thread safety guarantees of :class:`memoryview` objects. + If *exporter* is part of a chain or tree of buffer providers, two main schemes can be used: * Re-export: Each member of the tree acts as the exporting object and - sets :c:expr:`view->obj` to a new reference to itself. + sets ``view->obj`` to a new reference to itself. * Redirect: The buffer request is redirected to the root object of the - tree. Here, :c:expr:`view->obj` will be a new reference to the root + tree. Here, ``view->obj`` will be a new reference to the root object. The individual fields of *view* are described in section @@ -2836,6 +3028,8 @@ Buffer Object Structures .. c:member:: releasebufferproc PyBufferProcs.bf_releasebuffer + .. corresponding-type-slot:: Py_bf_releasebuffer + The signature of this function is:: void (PyObject *exporter, Py_buffer *view); @@ -2849,13 +3043,23 @@ Buffer Object Structures (2) If the counter is ``0``, free all memory associated with *view*. + **Thread safety:** + + In the :term:`free-threaded build`: + + * The export counter decrement in step (1) must be atomic. + + * Resource cleanup when the counter reaches zero must be done atomically, + as the final release may race with concurrent releases from other + threads and dellocation must only happen once. + The exporter MUST use the :c:member:`~Py_buffer.internal` field to keep track of buffer-specific resources. This field is guaranteed to remain constant, while a consumer MAY pass a copy of the original buffer as the *view* argument. - This function MUST NOT decrement :c:expr:`view->obj`, since that is + This function MUST NOT decrement ``view->obj``, since that is done automatically in :c:func:`PyBuffer_Release` (this scheme is useful for breaking reference cycles). @@ -2870,8 +3074,6 @@ Buffer Object Structures Async Object Structures ----------------------- -.. sectionauthor:: Yury Selivanov - .. versionadded:: 3.5 .. c:type:: PyAsyncMethods @@ -2890,6 +3092,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_await + .. corresponding-type-slot:: Py_am_await + The signature of this function is:: PyObject *am_await(PyObject *self); @@ -2901,6 +3105,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_aiter + .. corresponding-type-slot:: Py_am_aiter + The signature of this function is:: PyObject *am_aiter(PyObject *self); @@ -2913,6 +3119,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_anext + .. corresponding-type-slot:: Py_am_anext + The signature of this function is:: PyObject *am_anext(PyObject *self); @@ -2923,6 +3131,8 @@ Async Object Structures .. c:member:: sendfunc PyAsyncMethods.am_send + .. corresponding-type-slot:: Py_am_send + The signature of this function is:: PySendResult am_send(PyObject *self, PyObject *arg, PyObject **result); diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 84fee05cb4ce203..634dcbce7a57915 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -5,9 +5,6 @@ Unicode Objects and Codecs -------------------------- -.. sectionauthor:: Marc-André Lemburg -.. sectionauthor:: Georg Brandl - Unicode Objects ^^^^^^^^^^^^^^^ @@ -65,6 +62,27 @@ Python: .. versionadded:: 3.3 + The structure of a particular object can be determined using the following + macros. + The macros cannot fail; their behavior is undefined if their argument + is not a Python Unicode object. + + .. c:namespace:: NULL + + .. c:macro:: PyUnicode_IS_COMPACT(o) + + True if *o* uses the :c:struct:`PyCompactUnicodeObject` structure. + + .. versionadded:: 3.3 + + + .. c:macro:: PyUnicode_IS_COMPACT_ASCII(o) + + True if *o* uses the :c:struct:`PyASCIIObject` structure. + + .. versionadded:: 3.3 + + The following APIs are C macros and static inlined functions for fast checks and access to internal read-only data of Unicode objects: @@ -321,12 +339,22 @@ These APIs can be used to work with surrogates: Check if *ch* is a low surrogate (``0xDC00 <= ch <= 0xDFFF``). +.. c:function:: Py_UCS4 Py_UNICODE_HIGH_SURROGATE(Py_UCS4 ch) + + Return the high UTF-16 surrogate (``0xD800`` to ``0xDBFF``) for a Unicode + code point in the range ``[0x10000; 0x10FFFF]``. + +.. c:function:: Py_UCS4 Py_UNICODE_LOW_SURROGATE(Py_UCS4 ch) + + Return the low UTF-16 surrogate (``0xDC00`` to ``0xDFFF``) for a Unicode + code point in the range ``[0x10000; 0x10FFFF]``. + .. c:function:: Py_UCS4 Py_UNICODE_JOIN_SURROGATES(Py_UCS4 high, Py_UCS4 low) Join two surrogate code points and return a single :c:type:`Py_UCS4` value. *high* and *low* are respectively the leading and trailing surrogates in a - surrogate pair. *high* must be in the range [0xD800; 0xDBFF] and *low* must - be in the range [0xDC00; 0xDFFF]. + surrogate pair. *high* must be in the range ``[0xD800; 0xDBFF]`` and *low* must + be in the range ``[0xDC00; 0xDFFF]``. Creating and accessing Unicode strings @@ -734,7 +762,7 @@ APIs: The string must not have been “used” yet. See :c:func:`PyUnicode_New` for details. - Return the number of written character, or return ``-1`` and raise an + Return the number of written characters, or return ``-1`` and raise an exception on error. .. versionadded:: 3.3 @@ -747,7 +775,7 @@ APIs: Return ``0`` on success, ``-1`` on error with an exception set. This function checks that *unicode* is a Unicode object, that the index is - not out of bounds, and that the object's reference count is one). + not out of bounds, and that the object's reference count is one. See :c:func:`PyUnicode_WRITE` for a version that skips these checks, making them your responsibility. @@ -1146,7 +1174,7 @@ These are the UTF-8 codec APIs: .. versionadded:: 3.3 .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. + The return type is now ``const char *`` rather than ``char *``. .. versionchanged:: 3.10 This function is a part of the :ref:`limited API `. @@ -1168,7 +1196,7 @@ These are the UTF-8 codec APIs: .. versionadded:: 3.3 .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. + The return type is now ``const char *`` rather than ``char *``. UTF-32 Codecs @@ -1827,8 +1855,6 @@ object. On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. - .. versionadded:: 3.14 - .. c:function:: int PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, const wchar_t *str, Py_ssize_t size) Write the wide string *str* into *writer*. @@ -1839,7 +1865,7 @@ object. On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. -.. c:function:: int PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *writer, Py_UCS4 *str, Py_ssize_t size) +.. c:function:: int PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *writer, const Py_UCS4 *str, Py_ssize_t size) Writer the UCS4 string *str* into *writer*. @@ -1855,13 +1881,23 @@ object. On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. + To write a :class:`str` subclass which overrides the :meth:`~object.__str__` + method, :c:func:`PyUnicode_FromObject` can be used to get the original + string. + .. c:function:: int PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) Call :c:func:`PyObject_Repr` on *obj* and write the output into *writer*. + If *obj* is ``NULL``, write the string ``""`` into *writer*. + On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. + .. versionchanged:: 3.14.4 + + Added support for ``NULL``. + .. c:function:: int PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, Py_ssize_t start, Py_ssize_t end) Write the substring ``str[start:end]`` into *writer*. @@ -1917,7 +1953,7 @@ The following API is deprecated. whether you selected a "narrow" or "wide" Unicode version of Python at build time. - .. deprecated-removed:: 3.13 3.15 + .. deprecated-removed:: 3.13 3.16 .. c:function:: int PyUnicode_READY(PyObject *unicode) diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index fb07fec7effce8a..6256bf7a1454a9a 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -13,8 +13,9 @@ the interpreter. Several of these functions accept a start symbol from the grammar as a parameter. The available start symbols are :c:data:`Py_eval_input`, -:c:data:`Py_file_input`, and :c:data:`Py_single_input`. These are described -following the functions which accept them as parameters. +:c:data:`Py_file_input`, :c:data:`Py_single_input`, and +:c:data:`Py_func_type_input`. These are described following the functions +which accept them as parameters. Note also that several of these functions take :c:expr:`FILE*` parameters. One particular issue which needs to be handled carefully is that the :c:type:`FILE` @@ -99,18 +100,12 @@ the same library that the Python runtime is using. Otherwise, Python may not handle script file with LF line ending correctly. -.. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) - - This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, - leaving *flags* set to ``NULL``. - - -.. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) +.. c:function:: int PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) Read and execute a single statement from a file associated with an interactive device according to the *flags* argument. The user will be - prompted using ``sys.ps1`` and ``sys.ps2``. *filename* is decoded from the - :term:`filesystem encoding and error handler`. + prompted using ``sys.ps1`` and ``sys.ps2``. *filename* must be a Python + :class:`str` object. Returns ``0`` when the input was executed successfully, ``-1`` if there was an exception, or an error code @@ -119,6 +114,19 @@ the same library that the Python runtime is using. :file:`Python.h`, so must be included specifically if needed.) +.. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) + + This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, + leaving *flags* set to ``NULL``. + + +.. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) + + Similar to :c:func:`PyRun_InteractiveOneObject`, but *filename* is a + :c:expr:`const char*`, which is decoded from the + :term:`filesystem encoding and error handler`. + + .. c:function:: int PyRun_InteractiveLoop(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_InteractiveLoopFlags` below, @@ -140,7 +148,7 @@ the same library that the Python runtime is using. interpreter prompt is about to become idle and wait for user input from the terminal. The return value is ignored. Overriding this hook can be used to integrate the interpreter's prompt with other - event loops, as done in the :file:`Modules/_tkinter.c` in the + event loops, as done in :file:`Modules/_tkinter.c` in the Python source code. .. versionchanged:: 3.12 @@ -183,7 +191,7 @@ the same library that the Python runtime is using. objects *globals* and *locals* with the compiler flags specified by *flags*. *globals* must be a dictionary; *locals* can be any object that implements the mapping protocol. The parameter *start* specifies - the start token that should be used to parse the source code. + the start symbol and must be one of the :ref:`available start symbols `. Returns the result of executing the code as a Python object, or ``NULL`` if an exception was raised. @@ -231,9 +239,9 @@ the same library that the Python runtime is using. .. c:function:: PyObject* Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) Parse and compile the Python source code in *str*, returning the resulting code - object. The start token is given by *start*; this can be used to constrain the - code which can be compiled and should be :c:data:`Py_eval_input`, - :c:data:`Py_file_input`, or :c:data:`Py_single_input`. The filename specified by + object. The start symbol is given by *start*; this can be used to constrain the + code which can be compiled and should be :ref:`available start symbols + `. The filename specified by *filename* is used to construct the code object and may appear in tracebacks or :exc:`SyntaxError` exception messages. This returns ``NULL`` if the code cannot be parsed or compiled. @@ -296,32 +304,6 @@ the same library that the Python runtime is using. true on success, false on failure. -.. c:var:: int Py_eval_input - - .. index:: single: Py_CompileString (C function) - - The start symbol from the Python grammar for isolated expressions; for use with - :c:func:`Py_CompileString`. - - -.. c:var:: int Py_file_input - - .. index:: single: Py_CompileString (C function) - - The start symbol from the Python grammar for sequences of statements as read - from a file or other source; for use with :c:func:`Py_CompileString`. This is - the symbol to use when compiling arbitrarily long Python source code. - - -.. c:var:: int Py_single_input - - .. index:: single: Py_CompileString (C function) - - The start symbol from the Python grammar for a single statement; for use with - :c:func:`Py_CompileString`. This is the symbol used for the interactive - interpreter loop. - - .. c:struct:: PyCompilerFlags This is the structure used to hold compiler flags. In cases where code is only @@ -365,3 +347,92 @@ the same library that the Python runtime is using. as :c:macro:`CO_FUTURE_ANNOTATIONS` to enable features normally selectable using :ref:`future statements `. See :ref:`c_codeobject_flags` for a complete list. + + +.. _start-symbols: + +Available start symbols +^^^^^^^^^^^^^^^^^^^^^^^ + + +.. c:var:: int Py_eval_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for isolated expressions; for use with + :c:func:`Py_CompileString`. + + +.. c:var:: int Py_file_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for sequences of statements as read + from a file or other source; for use with :c:func:`Py_CompileString`. This is + the symbol to use when compiling arbitrarily long Python source code. + + +.. c:var:: int Py_single_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for a single statement; for use with + :c:func:`Py_CompileString`. This is the symbol used for the interactive + interpreter loop. + + +.. c:var:: int Py_func_type_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for a function type; for use with + :c:func:`Py_CompileString`. This is used to parse "signature type comments" + from :pep:`484`. + + This requires the :c:macro:`PyCF_ONLY_AST` flag to be set. + + .. seealso:: + * :py:class:`ast.FunctionType` + * :pep:`484` + + .. versionadded:: 3.8 + + +Stack Effects +^^^^^^^^^^^^^ + +.. seealso:: + :py:func:`dis.stack_effect` + + +.. c:macro:: PY_INVALID_STACK_EFFECT + + Sentinel value representing an invalid stack effect. + + This is currently equivalent to ``INT_MAX``. + + .. versionadded:: 3.8 + + +.. c:function:: int PyCompile_OpcodeStackEffect(int opcode, int oparg) + + Compute the stack effect of *opcode* with argument *oparg*. + + On success, this function returns the stack effect; on failure, this + returns :c:macro:`PY_INVALID_STACK_EFFECT`. + + .. versionadded:: 3.4 + + +.. c:function:: int PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump) + + Similar to :c:func:`PyCompile_OpcodeStackEffect`, but don't include the + stack effect of jumping if *jump* is zero. + + If *jump* is ``0``, this will not include the stack effect of jumping, but + if *jump* is ``1`` or ``-1``, this will include it. + + On success, this function returns the stack effect; on failure, this + returns :c:macro:`PY_INVALID_STACK_EFFECT`. + + .. versionadded:: 3.8 diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index c3c6cf413dcef58..8762a003c5218d3 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -19,7 +19,14 @@ as much as it can. .. c:function:: int PyWeakref_CheckRef(PyObject *ob) - Return non-zero if *ob* is a reference object. This function always succeeds. + Return non-zero if *ob* is a reference object or a subclass of the reference + type. This function always succeeds. + + +.. c:function:: int PyWeakref_CheckRefExact(PyObject *ob) + + Return non-zero if *ob* is a reference object, but not a subclass of the + reference type. This function always succeeds. .. c:function:: int PyWeakref_CheckProxy(PyObject *ob) @@ -36,7 +43,15 @@ as much as it can. should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a weakly referenceable object, or if *callback* is not callable, ``None``, or - ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + ``NULL``, this will raise :exc:`TypeError` and return ``NULL``. + + .. versionchanged:: next + Raise :exc:`!TypeError` if *callback* is not callable, ``None``, or + ``NULL``. + + .. seealso:: + :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly + referenceable. .. c:function:: PyObject* PyWeakref_NewProxy(PyObject *ob, PyObject *callback) @@ -48,7 +63,15 @@ as much as it can. collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a weakly referenceable object, or if *callback* is not callable, - ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + ``NULL``, this will raise :exc:`TypeError` and return ``NULL``. + + .. versionchanged:: next + Raise :exc:`!TypeError` if *callback* is not callable, ``None``, or + ``NULL``. + + .. seealso:: + :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly + referenceable. .. c:function:: int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) @@ -64,30 +87,6 @@ as much as it can. .. versionadded:: 3.13 -.. c:function:: PyObject* PyWeakref_GetObject(PyObject *ref) - - Return a :term:`borrowed reference` to the referenced object from a weak - reference, *ref*. If the referent is no longer live, returns ``Py_None``. - - .. note:: - - This function returns a :term:`borrowed reference` to the referenced object. - This means that you should always call :c:func:`Py_INCREF` on the object - except when it cannot be destroyed before the last usage of the borrowed - reference. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyWeakref_GetRef` instead. - - -.. c:function:: PyObject* PyWeakref_GET_OBJECT(PyObject *ref) - - Similar to :c:func:`PyWeakref_GetObject`, but does no error checking. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyWeakref_GetRef` instead. - - .. c:function:: int PyWeakref_IsDead(PyObject *ref) Test if the weak reference *ref* is dead. Returns 1 if the reference is diff --git a/Doc/conf.py b/Doc/conf.py index 1c1f36e5bc07371..9b103a594b235cf 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -8,15 +8,13 @@ import os import sys -from importlib import import_module from importlib.util import find_spec # Make our custom extensions available to Sphinx sys.path.append(os.path.abspath('tools/extensions')) sys.path.append(os.path.abspath('includes')) -# Python specific content from Doc/Tools/extensions/pyspecific.py -from pyspecific import SOURCE_URI +from patchlevel import get_header_version_info, get_version_info # General configuration # --------------------- @@ -33,6 +31,7 @@ 'issue_role', 'lexers', 'misc_news', + 'profiling_trace', 'pydoc_topics', 'pyspecific', 'sphinx.ext.coverage', @@ -42,8 +41,10 @@ # Skip if downstream redistributors haven't installed them _OPTIONAL_EXTENSIONS = ( + 'linklint.ext', 'notfound.extension', 'sphinxext.opengraph', + 'sphinxcontrib.rsvgconverter', ) for optional_ext in _OPTIONAL_EXTENSIONS: try: @@ -70,11 +71,12 @@ # General substitutions. project = 'Python' copyright = "2001 Python Software Foundation" +_doc_authors = 'Python documentation authors' # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. # See Doc/tools/extensions/patchlevel.py -version, release = import_module('patchlevel').get_version_info() +version, release = get_version_info() rst_epilog = f""" .. |python_version_literal| replace:: ``Python {version}`` @@ -174,6 +176,7 @@ ('c:type', '__int64'), ('c:type', 'unsigned __int64'), ('c:type', 'double'), + ('c:type', '_Float16'), # Standard C structures ('c:struct', 'in6_addr'), ('c:struct', 'in_addr'), @@ -221,31 +224,14 @@ ('envvar', 'USER'), ('envvar', 'USERNAME'), ('envvar', 'USERPROFILE'), - # Deprecated function that was never documented: - ('py:func', 'getargspec'), - ('py:func', 'inspect.getargspec'), - # Undocumented modules that users shouldn't have to worry about - # (implementation details of `os.path`): - ('py:mod', 'ntpath'), - ('py:mod', 'posixpath'), ] # Temporary undocumented names. # In future this list must be empty. nitpick_ignore += [ - # Undocumented public C macros - ('c:macro', 'Py_BUILD_ASSERT'), - ('c:macro', 'Py_BUILD_ASSERT_EXPR'), - # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot - # be resolved, as the method is currently undocumented. For context, see - # https://github.com/python/cpython/pull/103289. - ('py:meth', '_SubParsersAction.add_parser'), # Attributes/methods/etc. that definitely should be documented better, # but are deferred for now: - ('py:attr', '__annotations__'), - ('py:meth', '__missing__'), ('py:attr', '__wrapped__'), - ('py:meth', 'index'), # list.index, tuple.index, etc. ] # gh-106948: Copy standard C types declared in the "c:type" domain and C @@ -361,81 +347,85 @@ \sphinxstrong{Python Software Foundation}\\ Email: \sphinxemail{docs@python.org} } -\let\Verbatim=\OriginalVerbatim -\let\endVerbatim=\endOriginalVerbatim \setcounter{tocdepth}{2} ''', # The paper size ('letterpaper' or 'a4paper'). 'papersize': 'a4paper', # The font size ('10pt', '11pt' or '12pt'). 'pointsize': '10pt', + 'maxlistdepth': '8', # See https://github.com/python/cpython/issues/139588 } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). -_stdauthor = 'Guido van Rossum and the Python development team' latex_documents = [ - ('c-api/index', 'c-api.tex', 'The Python/C API', _stdauthor, 'manual'), + ('c-api/index', 'c-api.tex', 'The Python/C API', _doc_authors, 'manual'), ( 'extending/index', 'extending.tex', 'Extending and Embedding Python', - _stdauthor, + _doc_authors, 'manual', ), ( 'installing/index', 'installing.tex', 'Installing Python Modules', - _stdauthor, + _doc_authors, 'manual', ), ( 'library/index', 'library.tex', 'The Python Library Reference', - _stdauthor, + _doc_authors, 'manual', ), ( 'reference/index', 'reference.tex', 'The Python Language Reference', - _stdauthor, + _doc_authors, 'manual', ), ( 'tutorial/index', 'tutorial.tex', 'Python Tutorial', - _stdauthor, + _doc_authors, 'manual', ), ( 'using/index', 'using.tex', 'Python Setup and Usage', - _stdauthor, + _doc_authors, 'manual', ), ( 'faq/index', 'faq.tex', 'Python Frequently Asked Questions', - _stdauthor, + _doc_authors, 'manual', ), ( 'whatsnew/' + version, 'whatsnew.tex', 'What\'s New in Python', - 'A. M. Kuchling', + _doc_authors, 'howto', ), ] # Collect all HOWTOs individually latex_documents.extend( - ('howto/' + fn[:-4], 'howto-' + fn[:-4] + '.tex', '', _stdauthor, 'howto') + ( + 'howto/' + fn[:-4], + 'howto-' + fn[:-4] + '.tex', + '', + _doc_authors, + 'howto', + ) for fn in os.listdir('howto') if fn.endswith('.rst') and fn != 'index.rst' ) @@ -446,14 +436,38 @@ # Options for Epub output # ----------------------- -epub_author = 'Python Documentation Authors' +epub_author = _doc_authors epub_publisher = 'Python Software Foundation' -epub_exclude_files = ('index.xhtml', 'download.xhtml') +epub_exclude_files = ( + 'index.xhtml', + 'download.xhtml', + '_static/tachyon-example-flamegraph.html', + '_static/tachyon-example-heatmap.html', +) # index pages are not valid xhtml # https://github.com/sphinx-doc/sphinx/issues/12359 epub_use_index = False +# translation tag +# --------------- + +language_code = None +for arg in sys.argv: + if arg.startswith('language='): + language_code = arg.split('=', 1)[1] + +if language_code: + tags.add('translation') # noqa: F821 + + rst_epilog += f"""\ +.. _TRANSLATION_REPO: https://github.com/python/python-docs-{language_code.replace("_", "-").lower()} +""" # noqa: F821 +else: + rst_epilog += """\ +.. _TRANSLATION_REPO: https://github.com/python +""" + # Options for the coverage checker # -------------------------------- @@ -539,15 +553,20 @@ r'https://unix.org/version2/whatsnew/lp64_wp.html', ] + # Options for sphinx.ext.extlinks # ------------------------------- +v = get_header_version_info() +branch = "main" if v.releaselevel == "alpha" else f"{v.major}.{v.minor}" + # This config is a dictionary of external sites, # mapping unique short aliases to a base URL and a prefix. # https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html extlinks = { + "oss-fuzz": ("https://issues.oss-fuzz.com/issues/%s", "#%s"), "pypi": ("https://pypi.org/project/%s/", "%s"), - "source": (SOURCE_URI, "%s"), + "source": (f"https://github.com/python/cpython/tree/{branch}/%s", "%s"), } extlinks_detect_hardcoded_links = True @@ -557,6 +576,18 @@ # Relative filename of the data files refcount_file = 'data/refcounts.dat' stable_abi_file = 'data/stable_abi.dat' +threadsafety_file = 'data/threadsafety.dat' + +# Options for notfound.extension +# ------------------------------- + +if not os.getenv("READTHEDOCS"): + if language_code: + notfound_urls_prefix = ( + f'/{language_code.replace("_", "-").lower()}/{version}/' + ) + else: + notfound_urls_prefix = f'/{version}/' # Options for sphinxext-opengraph # ------------------------------- @@ -567,14 +598,11 @@ 'image': '_static/og-image.png', 'line_color': '#3776ab', } -if 'builder_html' in tags: # noqa: F821 - ogp_custom_meta_tags = [ - '', - ] - if 'create-social-cards' not in tags: # noqa: F821 - # Define a static preview image when not creating social cards - ogp_image = '_static/og-image.png' - ogp_custom_meta_tags += [ - '', - '', - ] +ogp_custom_meta_tags = ('',) +if 'create-social-cards' not in tags: # noqa: F821 + # Define a static preview image when not creating social cards + ogp_image = '_static/og-image.png' + ogp_custom_meta_tags += ( + '', + '', + ) diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 144c5608e074262..60c02aabeb89c51 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -1141,6 +1141,9 @@ PyInterpreterState_Clear:PyInterpreterState*:interp:: PyInterpreterState_Delete:void::: PyInterpreterState_Delete:PyInterpreterState*:interp:: +PyInterpreterState_GetDict:PyObject*::0: +PyInterpreterState_GetDict:PyInterpreterState*:interp:: + PyInterpreterState_GetID:int64_t::: PyInterpreterState_GetID:PyInterpreterState*:interp:: @@ -1469,6 +1472,9 @@ PyModule_Create2:PyObject*::+1: PyModule_Create2:PyModuleDef*:def:: PyModule_Create2:int:module_api_version:: +PyModule_Exec:int::: +PyModule_ExecDef:PyObject*:module:0: + PyModule_ExecDef:int::: PyModule_ExecDef:PyObject*:module:0: PyModule_ExecDef:PyModuleDef*:def:: @@ -1482,6 +1488,10 @@ PyModule_FromDefAndSpec2:PyModuleDef*:def:: PyModule_FromDefAndSpec2:PyObject*:spec:0: PyModule_FromDefAndSpec2:int:module_api_version:: +PyModule_FromSlotsAndSpec:PyObject*::+1: +PyModule_FromSlotsAndSpec:const PyModuleDef_Slot *:slots:: +PyModule_FromSlotsAndSpec:PyObject*:spec:0: + PyModule_GetDef:PyModuleDef*::0: PyModule_GetDef:PyObject*:module:0: @@ -1503,6 +1513,14 @@ PyModule_GetNameObject:PyObject*:module:0: PyModule_GetState:void*::: PyModule_GetState:PyObject*:module:0: +PyModule_GetStateSize:int::: +PyModule_GetStateSize:PyObject*:module:0: +PyModule_GetToken:Py_ssize_t**:result:: + +PyModule_GetToken:int::: +PyModule_GetToken:PyObject*:module:0: +PyModule_GetToken:void**:result:: + PyModule_New:PyObject*::+1: PyModule_New:char*:name:: @@ -2019,6 +2037,11 @@ PySeqIter_Check:PyObject *:op:0: PySeqIter_New:PyObject*::+1: PySeqIter_New:PyObject*:seq:0: +PySentinel_New:PyObject*::+1: +PySentinel_New:const char*:name:: +PySentinel_New:const char*:module_name:: +PySentinel_New:const char*:repr:: + PySequence_Check:int::: PySequence_Check:PyObject*:o:0: @@ -2409,6 +2432,20 @@ PyType_GetFlags:PyTypeObject*:type:0: PyType_GetName:PyObject*::+1: PyType_GetName:PyTypeObject*:type:0: +PyType_GetModule:PyObject*::0: +PyType_GetModule:PyTypeObject*:type:0: + +PyType_GetModule_DuringGC:PyObject*::0: +PyType_GetModule_DuringGC:PyTypeObject*:type:0: + +PyType_GetModuleByToken:PyObject*::+1: +PyType_GetModuleByToken:PyTypeObject*:type:0: +PyType_GetModuleByToken:PyModuleDef*:def:: + +PyType_GetModuleByToken_DuringGC:PyObject*::0: +PyType_GetModuleByToken_DuringGC:PyTypeObject*:type:0: +PyType_GetModuleByToken_DuringGC:PyModuleDef*:mod_token:: + PyType_GetModuleByDef:PyObject*::0: PyType_GetModuleByDef:PyTypeObject*:type:0: PyType_GetModuleByDef:PyModuleDef*:def:: @@ -2947,12 +2984,6 @@ PyWeakref_CheckProxy:PyObject*:ob:0: PyWeakref_CheckRef:int::: PyWeakref_CheckRef:PyObject*:ob:0: -PyWeakref_GET_OBJECT:PyObject*::0: -PyWeakref_GET_OBJECT:PyObject*:ref:0: - -PyWeakref_GetObject:PyObject*::0: -PyWeakref_GetObject:PyObject*:ref:0: - PyWeakref_GetRef:int::: PyWeakref_GetRef:PyObject*:ref:0: PyWeakref_GetRef:PyObject**:pobj:+1: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 0d0dfb3843260e5..86080fac7163838 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,5 +1,22 @@ role,name,added,ifdef_note,struct_abi_kind +macro,METH_CLASS,3.2,, +macro,METH_COEXIST,3.2,, +macro,METH_FASTCALL,3.10,, +macro,METH_METHOD,3.7,, +macro,METH_NOARGS,3.2,, +macro,METH_O,3.2,, +macro,METH_STATIC,3.2,, +macro,METH_VARARGS,3.2,, macro,PY_VECTORCALL_ARGUMENTS_OFFSET,3.12,, +type,PyABIInfo,3.15,,full-abi +func,PyABIInfo_Check,3.15,, +macro,PyABIInfo_DEFAULT_ABI_VERSION,3.15,, +macro,PyABIInfo_DEFAULT_FLAGS,3.15,, +macro,PyABIInfo_FREETHREADED,3.15,, +macro,PyABIInfo_FREETHREADING_AGNOSTIC,3.15,, +macro,PyABIInfo_GIL,3.15,, +macro,PyABIInfo_STABLE,3.15,, +macro,PyABIInfo_VAR,3.15,, func,PyAIter_Check,3.10,, func,PyArg_Parse,3.2,, func,PyArg_ParseTuple,3.2,, @@ -8,6 +25,26 @@ func,PyArg_UnpackTuple,3.2,, func,PyArg_VaParse,3.2,, func,PyArg_VaParseTupleAndKeywords,3.2,, func,PyArg_ValidateKeywordArguments,3.2,, +macro,PyBUF_ANY_CONTIGUOUS,3.11,, +macro,PyBUF_CONTIG,3.11,, +macro,PyBUF_CONTIG_RO,3.11,, +macro,PyBUF_C_CONTIGUOUS,3.11,, +macro,PyBUF_FORMAT,3.11,, +macro,PyBUF_FULL,3.11,, +macro,PyBUF_FULL_RO,3.11,, +macro,PyBUF_F_CONTIGUOUS,3.11,, +macro,PyBUF_INDIRECT,3.11,, +macro,PyBUF_MAX_NDIM,3.11,, +macro,PyBUF_ND,3.11,, +macro,PyBUF_READ,3.11,, +macro,PyBUF_RECORDS,3.11,, +macro,PyBUF_RECORDS_RO,3.11,, +macro,PyBUF_SIMPLE,3.11,, +macro,PyBUF_STRIDED,3.11,, +macro,PyBUF_STRIDED_RO,3.11,, +macro,PyBUF_STRIDES,3.11,, +macro,PyBUF_WRITABLE,3.11,, +macro,PyBUF_WRITE,3.11,, data,PyBaseObject_Type,3.2,, func,PyBool_FromLong,3.2,, data,PyBool_Type,3.2,, @@ -92,6 +129,12 @@ func,PyComplex_FromDoubles,3.2,, func,PyComplex_ImagAsDouble,3.2,, func,PyComplex_RealAsDouble,3.2,, data,PyComplex_Type,3.2,, +type,PyCriticalSection,3.15,,full-abi +type,PyCriticalSection2,3.15,,full-abi +func,PyCriticalSection2_Begin,3.15,, +func,PyCriticalSection2_End,3.15,, +func,PyCriticalSection_Begin,3.15,, +func,PyCriticalSection_End,3.15,, func,PyDescr_NewClassMethod,3.2,, func,PyDescr_NewGetSet,3.2,, func,PyDescr_NewMember,3.2,, @@ -123,6 +166,7 @@ func,PyDict_Merge,3.2,, func,PyDict_MergeFromSeq2,3.2,, func,PyDict_New,3.2,, func,PyDict_Next,3.2,, +func,PyDict_SetDefaultRef,3.15,, func,PyDict_SetItem,3.2,, func,PyDict_SetItemString,3.2,, func,PyDict_Size,3.2,, @@ -325,6 +369,10 @@ func,PyImport_ImportModuleLevel,3.2,, func,PyImport_ImportModuleLevelObject,3.7,, func,PyImport_ReloadModule,3.2,, func,PyIndex_Check,3.8,, +type,PyInterpreterGuard,3.15,,opaque +func,PyInterpreterGuard_Close,3.15,, +func,PyInterpreterGuard_FromCurrent,3.15,, +func,PyInterpreterGuard_FromView,3.15,, type,PyInterpreterState,3.2,,opaque func,PyInterpreterState_Clear,3.2,, func,PyInterpreterState_Delete,3.2,, @@ -332,6 +380,10 @@ func,PyInterpreterState_Get,3.9,, func,PyInterpreterState_GetDict,3.8,, func,PyInterpreterState_GetID,3.7,, func,PyInterpreterState_New,3.2,, +type,PyInterpreterView,3.15,,opaque +func,PyInterpreterView_Close,3.15,, +func,PyInterpreterView_FromCurrent,3.15,, +func,PyInterpreterView_FromMain,3.15,, func,PyIter_Check,3.8,, func,PyIter_Next,3.2,, func,PyIter_NextItem,3.14,, @@ -351,8 +403,14 @@ func,PyList_SetSlice,3.2,, func,PyList_Size,3.2,, func,PyList_Sort,3.2,, data,PyList_Type,3.2,, +type,PyLongExport,3.15,,full-abi +type,PyLongLayout,3.15,,full-abi type,PyLongObject,3.2,,opaque data,PyLongRangeIter_Type,3.2,, +type,PyLongWriter,3.15,,opaque +func,PyLongWriter_Create,3.15,, +func,PyLongWriter_Discard,3.15,, +func,PyLongWriter_Finish,3.15,, func,PyLong_AsDouble,3.2,, func,PyLong_AsInt,3.13,, func,PyLong_AsInt32,3.14,, @@ -371,6 +429,8 @@ func,PyLong_AsUnsignedLongLong,3.2,, func,PyLong_AsUnsignedLongLongMask,3.2,, func,PyLong_AsUnsignedLongMask,3.2,, func,PyLong_AsVoidPtr,3.2,, +func,PyLong_Export,3.15,, +func,PyLong_FreeExport,3.15,, func,PyLong_FromDouble,3.2,, func,PyLong_FromInt32,3.14,, func,PyLong_FromInt64,3.14,, @@ -387,7 +447,9 @@ func,PyLong_FromUnsignedLongLong,3.2,, func,PyLong_FromUnsignedNativeBytes,3.14,, func,PyLong_FromVoidPtr,3.2,, func,PyLong_GetInfo,3.2,, +func,PyLong_GetNativeLayout,3.15,, data,PyLong_Type,3.2,, +macro,PyMODEXPORT_FUNC,3.15,, data,PyMap_Type,3.2,, func,PyMapping_Check,3.2,, func,PyMapping_GetItemString,3.2,, @@ -422,9 +484,10 @@ func,PyMemoryView_GetContiguous,3.2,, data,PyMemoryView_Type,3.2,, type,PyMethodDef,3.2,,full-abi data,PyMethodDescr_Type,3.2,, -type,PyModuleDef,3.2,,full-abi -type,PyModuleDef_Base,3.2,,full-abi +type,PyModuleDef,3.2,,abi3t-opaque +type,PyModuleDef_Base,3.2,,abi3t-opaque func,PyModuleDef_Init,3.5,, +type,PyModuleDef_Slot,3.5,,full-abi data,PyModuleDef_Type,3.5,, func,PyModule_Add,3.13,, func,PyModule_AddFunctions,3.7,, @@ -434,8 +497,10 @@ func,PyModule_AddObjectRef,3.10,, func,PyModule_AddStringConstant,3.2,, func,PyModule_AddType,3.10,, func,PyModule_Create2,3.2,, +func,PyModule_Exec,3.15,, func,PyModule_ExecDef,3.7,, func,PyModule_FromDefAndSpec2,3.7,, +func,PyModule_FromSlotsAndSpec,3.15,, func,PyModule_GetDef,3.2,, func,PyModule_GetDict,3.2,, func,PyModule_GetFilename,3.2,, @@ -443,6 +508,10 @@ func,PyModule_GetFilenameObject,3.2,, func,PyModule_GetName,3.2,, func,PyModule_GetNameObject,3.7,, func,PyModule_GetState,3.2,, +func,PyModule_GetStateSize,3.15,, +func,PyModule_GetState_DuringGC,3.15,, +func,PyModule_GetToken,3.15,, +func,PyModule_GetToken_DuringGC,3.15,, func,PyModule_New,3.2,, func,PyModule_NewObject,3.7,, func,PyModule_SetDocString,3.7,, @@ -510,6 +579,7 @@ func,PyObject_ASCII,3.2,, func,PyObject_AsFileDescriptor,3.2,, func,PyObject_Bytes,3.2,, func,PyObject_Call,3.2,, +func,PyObject_CallFinalizerFromDealloc,3.15,, func,PyObject_CallFunction,3.2,, func,PyObject_CallFunctionObjArgs,3.2,, func,PyObject_CallMethod,3.2,, @@ -545,6 +615,7 @@ func,PyObject_GetIter,3.2,, func,PyObject_GetOptionalAttr,3.13,, func,PyObject_GetOptionalAttrString,3.13,, func,PyObject_GetTypeData,3.12,, +func,PyObject_GetTypeData_DuringGC,3.15,, func,PyObject_HasAttr,3.2,, func,PyObject_HasAttrString,3.2,, func,PyObject_HasAttrStringWithError,3.13,, @@ -613,6 +684,19 @@ func,PySlice_GetIndicesEx,3.2,, func,PySlice_New,3.2,, data,PySlice_Type,3.2,, func,PySlice_Unpack,3.7,, +type,PySlot,3.15,,full-abi +macro,PySlot_DATA,3.15,, +macro,PySlot_END,3.15,, +macro,PySlot_FUNC,3.15,, +macro,PySlot_INT64,3.15,, +macro,PySlot_INTPTR,3.15,, +macro,PySlot_OPTIONAL,3.15,, +macro,PySlot_PTR,3.15,, +macro,PySlot_PTR_STATIC,3.15,, +macro,PySlot_SIZE,3.15,, +macro,PySlot_STATIC,3.15,, +macro,PySlot_STATIC_DATA,3.15,, +macro,PySlot_UINT64,3.15,, func,PyState_AddModule,3.3,, func,PyState_FindModule,3.2,, func,PyState_RemoveModule,3.3,, @@ -634,21 +718,24 @@ func,PySys_GetObject,3.2,, func,PySys_GetOptionalAttr,3.15,, func,PySys_GetOptionalAttrString,3.15,, func,PySys_GetXOptions,3.7,, -func,PySys_ResetWarnOptions,3.2,, func,PySys_SetArgv,3.2,, func,PySys_SetArgvEx,3.2,, func,PySys_SetObject,3.2,, func,PySys_WriteStderr,3.2,, func,PySys_WriteStdout,3.2,, type,PyThreadState,3.2,,opaque +type,PyThreadStateToken,3.15,,opaque func,PyThreadState_Clear,3.2,, func,PyThreadState_Delete,3.2,, +func,PyThreadState_Ensure,3.15,, +func,PyThreadState_EnsureFromView,3.15,, func,PyThreadState_Get,3.2,, func,PyThreadState_GetDict,3.2,, func,PyThreadState_GetFrame,3.10,, func,PyThreadState_GetID,3.10,, func,PyThreadState_GetInterpreter,3.10,, func,PyThreadState_New,3.2,, +func,PyThreadState_Release,3.15,, func,PyThreadState_SetAsyncExc,3.2,, func,PyThreadState_Swap,3.2,, func,PyThread_GetInfo,3.3,, @@ -693,17 +780,23 @@ func,PyType_ClearCache,3.2,, func,PyType_Freeze,3.14,, func,PyType_FromMetaclass,3.12,, func,PyType_FromModuleAndSpec,3.10,, +func,PyType_FromSlots,3.15,, func,PyType_FromSpec,3.2,, func,PyType_FromSpecWithBases,3.3,, func,PyType_GenericAlloc,3.2,, func,PyType_GenericNew,3.2,, func,PyType_GetBaseByToken,3.14,, +func,PyType_GetBaseByToken_DuringGC,3.15,, func,PyType_GetFlags,3.2,, func,PyType_GetFullyQualifiedName,3.13,, func,PyType_GetModule,3.10,, func,PyType_GetModuleByDef,3.13,, +func,PyType_GetModuleByToken,3.15,, +func,PyType_GetModuleByToken_DuringGC,3.15,, func,PyType_GetModuleName,3.13,, func,PyType_GetModuleState,3.10,, +func,PyType_GetModuleState_DuringGC,3.15,, +func,PyType_GetModule_DuringGC,3.15,, func,PyType_GetName,3.11,, func,PyType_GetQualName,3.11,, func,PyType_GetSlot,3.4,, @@ -828,16 +921,25 @@ member,PyVarObject.ob_size,3.2,, func,PyVectorcall_Call,3.12,, func,PyVectorcall_NARGS,3.12,, type,PyWeakReference,3.2,,opaque -func,PyWeakref_GetObject,3.2,, func,PyWeakref_GetRef,3.13,, func,PyWeakref_NewProxy,3.2,, func,PyWeakref_NewRef,3.2,, data,PyWrapperDescr_Type,3.2,, func,PyWrapper_New,3.2,, data,PyZip_Type,3.2,, +macro,Py_ASNATIVEBYTES_ALLOW_INDEX,3.14,, +macro,Py_ASNATIVEBYTES_BIG_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_DEFAULTS,3.14,, +macro,Py_ASNATIVEBYTES_LITTLE_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_NATIVE_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_REJECT_NEGATIVE,3.14,, +macro,Py_ASNATIVEBYTES_UNSIGNED_BUFFER,3.14,, +macro,Py_AUDIT_READ,3.12,, func,Py_AddPendingCall,3.2,, func,Py_AtExit,3.2,, macro,Py_BEGIN_ALLOW_THREADS,3.2,, +macro,Py_BEGIN_CRITICAL_SECTION,3.15,, +macro,Py_BEGIN_CRITICAL_SECTION2,3.15,, macro,Py_BLOCK_THREADS,3.2,, func,Py_BuildValue,3.2,, func,Py_BytesMain,3.8,, @@ -845,6 +947,8 @@ func,Py_CompileString,3.2,, func,Py_DecRef,3.2,, func,Py_DecodeLocale,3.7,, macro,Py_END_ALLOW_THREADS,3.2,, +macro,Py_END_CRITICAL_SECTION,3.15,, +macro,Py_END_CRITICAL_SECTION2,3.15,, func,Py_EncodeLocale,3.7,, func,Py_EndInterpreter,3.2,, func,Py_EnterRecursiveCall,3.9,, @@ -865,6 +969,7 @@ func,Py_GetPlatform,3.2,, func,Py_GetRecursionLimit,3.2,, func,Py_GetVersion,3.2,, data,Py_HasFileSystemDefaultEncoding,3.2,, +func,Py_IS_TYPE,3.15,, func,Py_IncRef,3.2,, func,Py_Initialize,3.2,, func,Py_InitializeEx,3.2,, @@ -881,22 +986,159 @@ func,Py_NewInterpreter,3.2,, func,Py_NewRef,3.10,, func,Py_PACK_FULL_VERSION,3.14,, func,Py_PACK_VERSION,3.14,, +macro,Py_READONLY,3.12,, func,Py_REFCNT,3.14,, +macro,Py_RELATIVE_OFFSET,3.12,, func,Py_ReprEnter,3.2,, func,Py_ReprLeave,3.2,, +func,Py_SET_SIZE,3.15,, +func,Py_SIZE,3.15,, func,Py_SetProgramName,3.2,, func,Py_SetPythonHome,3.2,, func,Py_SetRecursionLimit,3.2,, +macro,Py_TPFLAGS_BASETYPE,3.2,, +macro,Py_TPFLAGS_DEFAULT,3.2,, +macro,Py_TPFLAGS_HAVE_GC,3.2,, +macro,Py_TPFLAGS_HAVE_VECTORCALL,3.12,, +macro,Py_TPFLAGS_ITEMS_AT_END,3.12,, +macro,Py_TPFLAGS_METHOD_DESCRIPTOR,3.8,, +macro,Py_TP_USE_SPEC,3.14,, func,Py_TYPE,3.14,, +macro,Py_T_BOOL,3.12,, +macro,Py_T_BYTE,3.12,, +macro,Py_T_CHAR,3.12,, +macro,Py_T_DOUBLE,3.12,, +macro,Py_T_FLOAT,3.12,, +macro,Py_T_INT,3.12,, +macro,Py_T_LONG,3.12,, +macro,Py_T_LONGLONG,3.12,, +macro,Py_T_OBJECT_EX,3.12,, +macro,Py_T_PYSSIZET,3.12,, +macro,Py_T_SHORT,3.12,, +macro,Py_T_STRING,3.12,, +macro,Py_T_STRING_INPLACE,3.12,, +macro,Py_T_UBYTE,3.12,, +macro,Py_T_UINT,3.12,, +macro,Py_T_ULONG,3.12,, +macro,Py_T_ULONGLONG,3.12,, +macro,Py_T_USHORT,3.12,, type,Py_UCS4,3.2,, macro,Py_UNBLOCK_THREADS,3.2,, data,Py_UTF8Mode,3.8,, func,Py_VaBuildValue,3.2,, data,Py_Version,3.11,, func,Py_XNewRef,3.10,, +macro,Py_am_aiter,3.5,, +macro,Py_am_anext,3.5,, +macro,Py_am_await,3.5,, +macro,Py_am_send,3.10,, +macro,Py_bf_getbuffer,3.11,, +macro,Py_bf_releasebuffer,3.11,, type,Py_buffer,3.11,,full-abi type,Py_intptr_t,3.2,, +macro,Py_mod_abi,3.15,, +macro,Py_mod_create,3.5,, +macro,Py_mod_doc,3.15,, +macro,Py_mod_exec,3.5,, +macro,Py_mod_gil,3.13,, +macro,Py_mod_methods,3.15,, +macro,Py_mod_multiple_interpreters,3.12,, +macro,Py_mod_name,3.15,, +macro,Py_mod_slots,3.15,, +macro,Py_mod_state_clear,3.15,, +macro,Py_mod_state_free,3.15,, +macro,Py_mod_state_size,3.15,, +macro,Py_mod_state_traverse,3.15,, +macro,Py_mod_token,3.15,, +macro,Py_mp_ass_subscript,3.2,, +macro,Py_mp_length,3.2,, +macro,Py_mp_subscript,3.2,, +macro,Py_nb_absolute,3.2,, +macro,Py_nb_add,3.2,, +macro,Py_nb_and,3.2,, +macro,Py_nb_bool,3.2,, +macro,Py_nb_divmod,3.2,, +macro,Py_nb_float,3.2,, +macro,Py_nb_floor_divide,3.2,, +macro,Py_nb_index,3.2,, +macro,Py_nb_inplace_add,3.2,, +macro,Py_nb_inplace_and,3.2,, +macro,Py_nb_inplace_floor_divide,3.2,, +macro,Py_nb_inplace_lshift,3.2,, +macro,Py_nb_inplace_matrix_multiply,3.5,, +macro,Py_nb_inplace_multiply,3.2,, +macro,Py_nb_inplace_or,3.2,, +macro,Py_nb_inplace_power,3.2,, +macro,Py_nb_inplace_remainder,3.2,, +macro,Py_nb_inplace_rshift,3.2,, +macro,Py_nb_inplace_subtract,3.2,, +macro,Py_nb_inplace_true_divide,3.2,, +macro,Py_nb_inplace_xor,3.2,, +macro,Py_nb_int,3.2,, +macro,Py_nb_invert,3.2,, +macro,Py_nb_lshift,3.2,, +macro,Py_nb_matrix_multiply,3.5,, +macro,Py_nb_multiply,3.2,, +macro,Py_nb_negative,3.2,, +macro,Py_nb_or,3.2,, +macro,Py_nb_positive,3.2,, +macro,Py_nb_power,3.2,, +macro,Py_nb_remainder,3.2,, +macro,Py_nb_rshift,3.2,, +macro,Py_nb_subtract,3.2,, +macro,Py_nb_true_divide,3.2,, +macro,Py_nb_xor,3.2,, +macro,Py_slot_end,3.15,, +macro,Py_slot_invalid,3.15,, +macro,Py_slot_subslots,3.15,, +macro,Py_sq_ass_item,3.2,, +macro,Py_sq_concat,3.2,, +macro,Py_sq_contains,3.2,, +macro,Py_sq_inplace_concat,3.2,, +macro,Py_sq_inplace_repeat,3.2,, +macro,Py_sq_item,3.2,, +macro,Py_sq_length,3.2,, +macro,Py_sq_repeat,3.2,, type,Py_ssize_t,3.2,, +macro,Py_tp_alloc,3.2,, +macro,Py_tp_base,3.2,, +macro,Py_tp_bases,3.2,, +macro,Py_tp_basicsize,3.15,, +macro,Py_tp_call,3.2,, +macro,Py_tp_clear,3.2,, +macro,Py_tp_dealloc,3.2,, +macro,Py_tp_del,3.2,, +macro,Py_tp_descr_get,3.2,, +macro,Py_tp_descr_set,3.2,, +macro,Py_tp_doc,3.2,, +macro,Py_tp_extra_basicsize,3.15,, +macro,Py_tp_finalize,3.5,, +macro,Py_tp_flags,3.15,, +macro,Py_tp_free,3.2,, +macro,Py_tp_getattr,3.2,, +macro,Py_tp_getattro,3.2,, +macro,Py_tp_getset,3.2,, +macro,Py_tp_hash,3.2,, +macro,Py_tp_init,3.2,, +macro,Py_tp_is_gc,3.2,, +macro,Py_tp_itemsize,3.15,, +macro,Py_tp_iter,3.2,, +macro,Py_tp_iternext,3.2,, +macro,Py_tp_members,3.2,, +macro,Py_tp_metaclass,3.15,, +macro,Py_tp_methods,3.2,, +macro,Py_tp_module,3.15,, +macro,Py_tp_name,3.15,, +macro,Py_tp_new,3.2,, +macro,Py_tp_repr,3.2,, +macro,Py_tp_richcompare,3.2,, +macro,Py_tp_setattr,3.2,, +macro,Py_tp_setattro,3.2,, +macro,Py_tp_slots,3.15,, +macro,Py_tp_str,3.2,, +macro,Py_tp_token,3.14,, +macro,Py_tp_traverse,3.2,, +macro,Py_tp_vectorcall,3.14,, type,Py_uintptr_t,3.2,, type,allocfunc,3.2,, type,binaryfunc,3.2,, diff --git a/Doc/data/threadsafety.dat b/Doc/data/threadsafety.dat new file mode 100644 index 000000000000000..ea5a24a5505e208 --- /dev/null +++ b/Doc/data/threadsafety.dat @@ -0,0 +1,284 @@ +# Thread safety annotations for C API functions. +# +# Each line has the form: +# function_name : level +# +# Where level is one of: +# incompatible -- not safe even with external locking +# compatible -- safe if the caller serializes all access with external locks +# distinct -- safe on distinct objects without external synchronization +# shared -- safe for concurrent use on the same object +# atomic -- atomic +# +# Lines beginning with '#' are ignored. +# The function name must match the C domain identifier used in the documentation. + +# Synchronization primitives (Doc/c-api/synchronization.rst) +PyMutex_Lock:atomic: +PyMutex_Unlock:atomic: +PyMutex_IsLocked:atomic: + + +# Dictionary objects (Doc/c-api/dict.rst) + +# Type checks - read ob_type pointer, always safe +PyDict_Check:atomic: +PyDict_CheckExact:atomic: + +# Creation - pure allocation, no shared state +PyDict_New:atomic: + +# Lock-free lookups - use _Py_dict_lookup_threadsafe(), no locking. +# Atomic with simple types. +PyDict_Contains:shared: +PyDict_ContainsString:atomic: +PyDict_GetItemRef:shared: +PyDict_GetItemStringRef:atomic: +PyDict_Size:atomic: +PyDict_GET_SIZE:atomic: + +# Borrowed-reference lookups - lock-free dict access but returned +# borrowed reference is unsafe in free-threaded builds without +# external synchronization +PyDict_GetItem:compatible: +PyDict_GetItemWithError:compatible: +PyDict_GetItemString:compatible: +PyDict_SetDefault:compatible: + +# Iteration - no locking; returns borrowed refs +PyDict_Next:compatible: + +# Single-item mutations - protected by per-object critical section +PyDict_SetItem:shared: +PyDict_SetItemString:atomic: +PyDict_DelItem:shared: +PyDict_DelItemString:atomic: +PyDict_SetDefaultRef:shared: +PyDict_Pop:shared: +PyDict_PopString:atomic: + +# Bulk reads - hold per-object lock for duration +PyDict_Clear:atomic: +PyDict_Copy:atomic: +PyDict_Keys:atomic: +PyDict_Values:atomic: +PyDict_Items:atomic: + +# Merge/update - lock target dict; also lock source when it is a dict +PyDict_Update:shared: +PyDict_Merge:shared: +PyDict_MergeFromSeq2:shared: + +# Watcher registration - no synchronization on interpreter state +PyDict_AddWatcher:compatible: +PyDict_ClearWatcher:compatible: + +# Per-dict watcher tags - non-atomic RMW on _ma_watcher_tag; +# safe on distinct dicts only +PyDict_Watch:distinct: +PyDict_Unwatch:distinct: + + +# List objects (Doc/c-api/list.rst) + +# Type checks - read ob_type pointer, always safe +PyList_Check:atomic: +PyList_CheckExact:atomic: + +# Creation - pure allocation, no shared state +PyList_New:atomic: + +# Size - uses atomic load on free-threaded builds +PyList_Size:atomic: +PyList_GET_SIZE:atomic: + +# Strong-reference lookup - lock-free with atomic ops +PyList_GetItemRef:atomic: + +# Borrowed-reference lookups - no locking; returned borrowed +# reference is unsafe in free-threaded builds without +# external synchronization +PyList_GetItem:compatible: +PyList_GET_ITEM:compatible: + +# Single-item mutations - hold per-object lock for duration; +# appear atomic to lock-free readers +PyList_SetItem:atomic: +PyList_Append:atomic: + +# Insert - protected by per-object critical section; shifts +# elements so lock-free readers may observe intermediate states +PyList_Insert:shared: + +# Initialization macro - no synchronization; normally only used +# to fill in new lists where there is no previous content +PyList_SET_ITEM:compatible: + +# Bulk operations - hold per-object lock for duration +PyList_GetSlice:atomic: +PyList_AsTuple:atomic: +PyList_Clear:atomic: + +# Reverse - protected by per-object critical section; swaps +# elements so lock-free readers may observe intermediate states +PyList_Reverse:shared: + +# Slice assignment - lock target list; also lock source when it +# is a list +PyList_SetSlice:shared: + +# Sort - per-object lock held; the list is emptied before sorting +# so other threads may observe an empty list, but they won't see the +# intermediate states of the sort +PyList_Sort:shared: + +# Extend - lock target list; also lock source when it is a +# list, set, or dict +PyList_Extend:shared: + +# Creation - pure allocation, no shared state +PyBytes_FromString:atomic: +PyBytes_FromStringAndSize:atomic: +PyBytes_DecodeEscape:atomic: + +# Creation from formatting C primitives - pure allocation, no shared state +PyBytes_FromFormat:atomic: +PyBytes_FromFormatV:atomic: + +# Creation from object - uses buffer protocol so may call arbitrary code; +# safe as long as the buffer is not mutated by another thread during the operation +PyBytes_FromObject:shared: + +# Size - uses atomic load on free-threaded builds +PyBytes_Size:atomic: +PyBytes_GET_SIZE:atomic: + +# Raw data - no locking; mutating it is unsafe if the bytes object is shared between threads +PyBytes_AsString:compatible: +PyBytes_AS_STRING:compatible: +PyBytes_AsStringAndSize:compatible: + +# Concatenation - uses buffer protocol; safe as long as buffer is not mutated by another thread during the operation +PyBytes_Concat:shared: +PyBytes_ConcatAndDel:shared: +PyBytes_Join:shared: + +# Resizing - safe if the object is unique +_PyBytes_Resize:distinct: + +# Repr - atomic as bytes are immutable +PyBytes_Repr:atomic: + +# Creation from object - may call arbitrary code +PyByteArray_FromObject:shared: + +# Creation - pure allocation, no shared state +PyByteArray_FromStringAndSize:atomic: + +# Concatenation - uses buffer protocol; safe as long as buffer is not mutated by another thread during the operation +PyByteArray_Concat:shared: + +# Size - uses atomic load on free-threaded builds +PyByteArray_Size:atomic: +PyByteArray_GET_SIZE:atomic: + +# Raw data - no locking; mutating it is unsafe if the bytearray object is shared between threads +PyByteArray_AsString:compatible: +PyByteArray_AS_STRING:compatible: + +# Creation - may iterate the iterable argument, calling arbitrary code. +# Atomic for sets, frozensets, dicts, and frozendicts. +PySet_New:shared: +PyFrozenSet_New:shared: + +# Size - uses atomic load on free-threaded builds +PySet_Size:atomic: +PySet_GET_SIZE:atomic: + +# Contains - lock-free, atomic with simple types +PySet_Contains:shared: + +# Mutations - hold per-object lock for duration +# atomic with simple types +PySet_Add:shared: +PySet_Discard:shared: + +# Pop - hold per-object lock for duration +PySet_Pop:atomic: + +# Clear - empties the set before clearing +PySet_Clear:atomic: + +# Capsule objects (Doc/c-api/capsule.rst) + +# Type check - read ob_type pointer, always safe +PyCapsule_CheckExact:atomic: + +# Creation - pure allocation, no shared state +PyCapsule_New:atomic: + +# Validation - reads pointer and name fields; safe on distinct objects +PyCapsule_IsValid:distinct: + +# Getters - read struct fields; safe on distinct objects but +# concurrent access to the same capsule requires external synchronization +PyCapsule_GetPointer:distinct: +PyCapsule_GetName:distinct: +PyCapsule_GetDestructor:distinct: +PyCapsule_GetContext:distinct: + +# Setters - write struct fields; safe on distinct objects but +# concurrent access to the same capsule requires external synchronization +PyCapsule_SetPointer:distinct: +PyCapsule_SetName:distinct: +PyCapsule_SetDestructor:distinct: +PyCapsule_SetContext:distinct: + +# Import - looks up a capsule from a module attribute and +# calls PyCapsule_GetPointer; may call arbitrary code +PyCapsule_Import:compatible: + +# Tuple objects + +# Creation - pure allocation, no shared state +PyTuple_New:atomic: +PyTuple_FromArray:atomic: +PyTuple_Pack:atomic: + +# Size - tuples are immutable so size never changes +PyTuple_Size:atomic: +PyTuple_GET_SIZE:atomic: + +# Borrowed-reference lookups - tuples are immutable so items +# never change, however the tuple must be kept alive while using the borrowed reference +PyTuple_GetItem:compatible: +PyTuple_GET_ITEM:compatible: + +# Slice - creates a new tuple from an existing tuple +PyTuple_GetSlice:atomic: + +# SetItem - only usable on tuples with refcount 1 +PyTuple_SetItem:compatible: +PyTuple_SET_ITEM:compatible: + +# Resize - only usable on tuples with refcount 1 +_PyTuple_Resize:compatible: + +# Struct Sequence objects + +# Creation +PyStructSequence_NewType:atomic: +PyStructSequence_New:atomic: + +# Initialization - modifies the type object in place +PyStructSequence_InitType:distinct: +PyStructSequence_InitType2:distinct: + +# Borrowed-reference lookups - same as tuple items +PyStructSequence_GetItem:compatible: +PyStructSequence_GET_ITEM:compatible: + +# SetItem - only for filling in brand new instances +PyStructSequence_SetItem:compatible: +PyStructSequence_SET_ITEM:compatible: + diff --git a/Doc/deprecations/c-api-pending-removal-in-3.15.rst b/Doc/deprecations/c-api-pending-removal-in-3.15.rst index a3e335ecaf43244..6d0d47403ff2ed4 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.15.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.15.rst @@ -3,12 +3,10 @@ Pending removal in Python 3.15 * The :c:func:`!PyImport_ImportModuleNoBlock`: Use :c:func:`PyImport_ImportModule` instead. -* :c:func:`PyWeakref_GetObject` and :c:func:`PyWeakref_GET_OBJECT`: +* :c:func:`!PyWeakref_GetObject` and :c:func:`!PyWeakref_GET_OBJECT`: Use :c:func:`PyWeakref_GetRef` instead. The `pythoncapi-compat project `__ can be used to get :c:func:`PyWeakref_GetRef` on Python 3.12 and older. -* :c:type:`Py_UNICODE` type and the :c:macro:`!Py_UNICODE_WIDE` macro: - Use :c:type:`wchar_t` instead. * :c:func:`!PyUnicode_AsDecodedObject`: Use :c:func:`PyCodec_Decode` instead. * :c:func:`!PyUnicode_AsDecodedUnicode`: @@ -48,88 +46,3 @@ Pending removal in Python 3.15 The `pythoncapi-compat project `__ can be used to get :c:func:`PyConfig_Get` on Python 3.13 and older. - -* Functions to configure Python's initialization, deprecated in Python 3.11: - - * :c:func:`!PySys_SetArgvEx()`: - Set :c:member:`PyConfig.argv` instead. - * :c:func:`!PySys_SetArgv()`: - Set :c:member:`PyConfig.argv` instead. - * :c:func:`!Py_SetProgramName()`: - Set :c:member:`PyConfig.program_name` instead. - * :c:func:`!Py_SetPythonHome()`: - Set :c:member:`PyConfig.home` instead. - * :c:func:`PySys_ResetWarnOptions`: - Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. - - The :c:func:`Py_InitializeFromConfig` API should be used with - :c:type:`PyConfig` instead. - -* Global configuration variables: - - * :c:var:`Py_DebugFlag`: - Use :c:member:`PyConfig.parser_debug` or - :c:func:`PyConfig_Get("parser_debug") ` instead. - * :c:var:`Py_VerboseFlag`: - Use :c:member:`PyConfig.verbose` or - :c:func:`PyConfig_Get("verbose") ` instead. - * :c:var:`Py_QuietFlag`: - Use :c:member:`PyConfig.quiet` or - :c:func:`PyConfig_Get("quiet") ` instead. - * :c:var:`Py_InteractiveFlag`: - Use :c:member:`PyConfig.interactive` or - :c:func:`PyConfig_Get("interactive") ` instead. - * :c:var:`Py_InspectFlag`: - Use :c:member:`PyConfig.inspect` or - :c:func:`PyConfig_Get("inspect") ` instead. - * :c:var:`Py_OptimizeFlag`: - Use :c:member:`PyConfig.optimization_level` or - :c:func:`PyConfig_Get("optimization_level") ` instead. - * :c:var:`Py_NoSiteFlag`: - Use :c:member:`PyConfig.site_import` or - :c:func:`PyConfig_Get("site_import") ` instead. - * :c:var:`Py_BytesWarningFlag`: - Use :c:member:`PyConfig.bytes_warning` or - :c:func:`PyConfig_Get("bytes_warning") ` instead. - * :c:var:`Py_FrozenFlag`: - Use :c:member:`PyConfig.pathconfig_warnings` or - :c:func:`PyConfig_Get("pathconfig_warnings") ` instead. - * :c:var:`Py_IgnoreEnvironmentFlag`: - Use :c:member:`PyConfig.use_environment` or - :c:func:`PyConfig_Get("use_environment") ` instead. - * :c:var:`Py_DontWriteBytecodeFlag`: - Use :c:member:`PyConfig.write_bytecode` or - :c:func:`PyConfig_Get("write_bytecode") ` instead. - * :c:var:`Py_NoUserSiteDirectory`: - Use :c:member:`PyConfig.user_site_directory` or - :c:func:`PyConfig_Get("user_site_directory") ` instead. - * :c:var:`Py_UnbufferedStdioFlag`: - Use :c:member:`PyConfig.buffered_stdio` or - :c:func:`PyConfig_Get("buffered_stdio") ` instead. - * :c:var:`Py_HashRandomizationFlag`: - Use :c:member:`PyConfig.use_hash_seed` - and :c:member:`PyConfig.hash_seed` or - :c:func:`PyConfig_Get("hash_seed") ` instead. - * :c:var:`Py_IsolatedFlag`: - Use :c:member:`PyConfig.isolated` or - :c:func:`PyConfig_Get("isolated") ` instead. - * :c:var:`Py_LegacyWindowsFSEncodingFlag`: - Use :c:member:`PyPreConfig.legacy_windows_fs_encoding` or - :c:func:`PyConfig_Get("legacy_windows_fs_encoding") ` instead. - * :c:var:`Py_LegacyWindowsStdioFlag`: - Use :c:member:`PyConfig.legacy_windows_stdio` or - :c:func:`PyConfig_Get("legacy_windows_stdio") ` instead. - * :c:var:`!Py_FileSystemDefaultEncoding`, :c:var:`!Py_HasFileSystemDefaultEncoding`: - Use :c:member:`PyConfig.filesystem_encoding` or - :c:func:`PyConfig_Get("filesystem_encoding") ` instead. - * :c:var:`!Py_FileSystemDefaultEncodeErrors`: - Use :c:member:`PyConfig.filesystem_errors` or - :c:func:`PyConfig_Get("filesystem_errors") ` instead. - * :c:var:`!Py_UTF8Mode`: - Use :c:member:`PyPreConfig.utf8_mode` or - :c:func:`PyConfig_Get("utf8_mode") ` instead. - (see :c:func:`Py_PreInitialize`) - - The :c:func:`Py_InitializeFromConfig` API should be used with - :c:type:`PyConfig` to set these options. Or :c:func:`PyConfig_Get` can be - used to get these options at runtime. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.16.rst b/Doc/deprecations/c-api-pending-removal-in-3.16.rst index 9453f83799c43d7..fe2d91cf316b18f 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.16.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.16.rst @@ -1,4 +1,87 @@ Pending removal in Python 3.16 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* The bundled copy of ``libmpdec``. +* Functions to configure Python's initialization, deprecated in Python 3.11: + + * :c:func:`!PySys_SetArgvEx()`: + Set :c:member:`PyConfig.argv` instead. + * :c:func:`!PySys_SetArgv()`: + Set :c:member:`PyConfig.argv` instead. + * :c:func:`!Py_SetProgramName()`: + Set :c:member:`PyConfig.program_name` instead. + * :c:func:`!Py_SetPythonHome()`: + Set :c:member:`PyConfig.home` instead. + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` instead. + +* Global configuration variables: + + * :c:var:`Py_DebugFlag`: + Use :c:member:`PyConfig.parser_debug` or + :c:func:`PyConfig_Get("parser_debug") ` instead. + * :c:var:`Py_VerboseFlag`: + Use :c:member:`PyConfig.verbose` or + :c:func:`PyConfig_Get("verbose") ` instead. + * :c:var:`Py_QuietFlag`: + Use :c:member:`PyConfig.quiet` or + :c:func:`PyConfig_Get("quiet") ` instead. + * :c:var:`Py_InteractiveFlag`: + Use :c:member:`PyConfig.interactive` or + :c:func:`PyConfig_Get("interactive") ` instead. + * :c:var:`Py_InspectFlag`: + Use :c:member:`PyConfig.inspect` or + :c:func:`PyConfig_Get("inspect") ` instead. + * :c:var:`Py_OptimizeFlag`: + Use :c:member:`PyConfig.optimization_level` or + :c:func:`PyConfig_Get("optimization_level") ` instead. + * :c:var:`Py_NoSiteFlag`: + Use :c:member:`PyConfig.site_import` or + :c:func:`PyConfig_Get("site_import") ` instead. + * :c:var:`Py_BytesWarningFlag`: + Use :c:member:`PyConfig.bytes_warning` or + :c:func:`PyConfig_Get("bytes_warning") ` instead. + * :c:var:`Py_FrozenFlag`: + Use :c:member:`PyConfig.pathconfig_warnings` or + :c:func:`PyConfig_Get("pathconfig_warnings") ` instead. + * :c:var:`Py_IgnoreEnvironmentFlag`: + Use :c:member:`PyConfig.use_environment` or + :c:func:`PyConfig_Get("use_environment") ` instead. + * :c:var:`Py_DontWriteBytecodeFlag`: + Use :c:member:`PyConfig.write_bytecode` or + :c:func:`PyConfig_Get("write_bytecode") ` instead. + * :c:var:`Py_NoUserSiteDirectory`: + Use :c:member:`PyConfig.user_site_directory` or + :c:func:`PyConfig_Get("user_site_directory") ` instead. + * :c:var:`Py_UnbufferedStdioFlag`: + Use :c:member:`PyConfig.buffered_stdio` or + :c:func:`PyConfig_Get("buffered_stdio") ` instead. + * :c:var:`Py_HashRandomizationFlag`: + Use :c:member:`PyConfig.use_hash_seed` + and :c:member:`PyConfig.hash_seed` or + :c:func:`PyConfig_Get("hash_seed") ` instead. + * :c:var:`Py_IsolatedFlag`: + Use :c:member:`PyConfig.isolated` or + :c:func:`PyConfig_Get("isolated") ` instead. + * :c:var:`Py_LegacyWindowsFSEncodingFlag`: + Use :c:member:`PyPreConfig.legacy_windows_fs_encoding` or + :c:func:`PyConfig_Get("legacy_windows_fs_encoding") ` instead. + * :c:var:`Py_LegacyWindowsStdioFlag`: + Use :c:member:`PyConfig.legacy_windows_stdio` or + :c:func:`PyConfig_Get("legacy_windows_stdio") ` instead. + * :c:var:`!Py_FileSystemDefaultEncoding`, :c:var:`!Py_HasFileSystemDefaultEncoding`: + Use :c:member:`PyConfig.filesystem_encoding` or + :c:func:`PyConfig_Get("filesystem_encoding") ` instead. + * :c:var:`!Py_FileSystemDefaultEncodeErrors`: + Use :c:member:`PyConfig.filesystem_errors` or + :c:func:`PyConfig_Get("filesystem_errors") ` instead. + * :c:var:`!Py_UTF8Mode`: + Use :c:member:`PyPreConfig.utf8_mode` or + :c:func:`PyConfig_Get("utf8_mode") ` instead. + (see :c:func:`Py_PreInitialize`) + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` to set these options. Or :c:func:`PyConfig_Get` can be + used to get these options at runtime. + +* :c:type:`Py_UNICODE` which was deprecated by :pep:`393`. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.18.rst b/Doc/deprecations/c-api-pending-removal-in-3.18.rst index 564cfb79a0b5134..820334ee43c82e4 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.18.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.18.rst @@ -1,11 +1,12 @@ Pending removal in Python 3.18 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* Deprecated private functions (:gh:`128863`): +* The following private functions are deprecated + and planned for removal in Python 3.18: * :c:func:`!_PyBytes_Join`: use :c:func:`PyBytes_Join`. * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. - * :c:func:`!_PyDict_Pop()`: :c:func:`PyDict_Pop`. + * :c:func:`!_PyDict_Pop()`: use :c:func:`PyDict_Pop`. * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. * :c:func:`!_PyLong_FromDigits` and :c:func:`!_PyLong_New`: use :c:func:`PyLongWriter_Create`. @@ -31,7 +32,7 @@ Pending removal in Python 3.18 :c:func:`PyUnicodeWriter_WriteSubstring(writer, str, start, end) `. * :c:func:`!_PyUnicodeWriter_WriteASCIIString`: replace ``_PyUnicodeWriter_WriteASCIIString(&writer, str)`` with - :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) `. + :c:func:`PyUnicodeWriter_WriteASCII(writer, str) `. * :c:func:`!_PyUnicodeWriter_WriteLatin1String`: replace ``_PyUnicodeWriter_WriteLatin1String(&writer, str)`` with :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) `. @@ -39,7 +40,12 @@ Pending removal in Python 3.18 * :c:func:`!_PyUnicodeWriter_PrepareKind`: (no replacement). * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. * :c:func:`!_Py_fopen_obj`: use :c:func:`Py_fopen`. + * :c:func:`PyGen_New`: (no replacement). + * :c:func:`PyGen_NewWithQualName`: (no replacement). + * :c:func:`PyCoro_New`: (no replacement). + * :c:func:`PyAsyncGen_New`: (no replacement). The `pythoncapi-compat project - `__ can be used to get these - new public functions on Python 3.13 and older. + `__ can be used to get + these new public functions on Python 3.13 and older. + (Contributed by Victor Stinner in :gh:`128863`.) diff --git a/Doc/deprecations/c-api-pending-removal-in-3.19.rst b/Doc/deprecations/c-api-pending-removal-in-3.19.rst new file mode 100644 index 000000000000000..ac9dcb8b424a17d --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.19.rst @@ -0,0 +1,4 @@ +Pending removal in Python 3.19 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* :pep:`456` embedders support for the string hashing scheme definition. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.20.rst b/Doc/deprecations/c-api-pending-removal-in-3.20.rst new file mode 100644 index 000000000000000..8de55bbe7e695c3 --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.20.rst @@ -0,0 +1,16 @@ +Pending removal in Python 3.20 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* :c:func:`!_PyObject_CallMethodId`, :c:func:`!_PyObject_GetAttrId` and + :c:func:`!_PyUnicode_FromId` are deprecated since 3.15 and will be removed in + 3.20. Instead, use :c:func:`PyUnicode_InternFromString()` and cache the result in + the module state, then call :c:func:`PyObject_CallMethod` or + :c:func:`PyObject_GetAttr`. + (Contributed by Victor Stinner in :gh:`141049`.) + +* The ``cval`` field in :c:type:`PyComplexObject` (:gh:`128813`). + Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex` + to convert a Python complex number to/from the C :c:type:`Py_complex` + representation. + +* Macros :c:macro:`!Py_MATH_PIl` and :c:macro:`!Py_MATH_El`. diff --git a/Doc/deprecations/index.rst b/Doc/deprecations/index.rst index d064f2bec42c229..0ac0a0289d5fd40 100644 --- a/Doc/deprecations/index.rst +++ b/Doc/deprecations/index.rst @@ -1,21 +1,31 @@ Deprecations ============ -.. include:: pending-removal-in-3.15.rst - .. include:: pending-removal-in-3.16.rst .. include:: pending-removal-in-3.17.rst +.. include:: pending-removal-in-3.18.rst + .. include:: pending-removal-in-3.19.rst +.. include:: pending-removal-in-3.20.rst + +.. include:: pending-removal-in-3.21.rst + .. include:: pending-removal-in-future.rst +.. include:: soft-deprecations.rst + C API deprecations ------------------ -.. include:: c-api-pending-removal-in-3.15.rst +.. include:: c-api-pending-removal-in-3.16.rst .. include:: c-api-pending-removal-in-3.18.rst +.. include:: c-api-pending-removal-in-3.19.rst + +.. include:: c-api-pending-removal-in-3.20.rst + .. include:: c-api-pending-removal-in-future.rst diff --git a/Doc/deprecations/pending-removal-in-3.13.rst b/Doc/deprecations/pending-removal-in-3.13.rst index 2fd2f12cc6a2c42..d5b8c80e8f9aa0f 100644 --- a/Doc/deprecations/pending-removal-in-3.13.rst +++ b/Doc/deprecations/pending-removal-in-3.13.rst @@ -38,15 +38,3 @@ APIs: * :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) * :class:`!webbrowser.MacOSX` (:gh:`86421`) * :class:`classmethod` descriptor chaining (:gh:`89519`) -* :mod:`importlib.resources` deprecated methods: - - * ``contents()`` - * ``is_resource()`` - * ``open_binary()`` - * ``open_text()`` - * ``path()`` - * ``read_binary()`` - * ``read_text()`` - - Use :func:`importlib.resources.files` instead. Refer to `importlib-resources: Migrating from Legacy - `_ (:gh:`106531`) diff --git a/Doc/deprecations/pending-removal-in-3.14.rst b/Doc/deprecations/pending-removal-in-3.14.rst index 9aac10840a663ff..171758156ab1176 100644 --- a/Doc/deprecations/pending-removal-in-3.14.rst +++ b/Doc/deprecations/pending-removal-in-3.14.rst @@ -38,12 +38,6 @@ Pending removal in Python 3.14 is no current event loop set and it decides to create one. (Contributed by Serhiy Storchaka and Guido van Rossum in :gh:`100160`.) -* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. - Prefer :class:`!Sequence` or :class:`~collections.abc.Buffer`. - For use in typing, prefer a union, like ``bytes | bytearray``, - or :class:`collections.abc.Buffer`. - (Contributed by Shantanu Jain in :gh:`91896`.) - * :mod:`email`: Deprecated the *isdst* parameter in :func:`email.utils.localtime`. (Contributed by Alan Williams in :gh:`72346`.) @@ -96,9 +90,6 @@ Pending removal in Python 3.14 if :ref:`named placeholders ` are used and *parameters* is a sequence instead of a :class:`dict`. -* :mod:`typing`: :class:`!typing.ByteString`, deprecated since Python 3.9, - now causes a :exc:`DeprecationWarning` to be emitted when it is used. - * :mod:`urllib`: :class:`!urllib.parse.Quoter` is deprecated: it was not intended to be a public API. diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst index 9505bcfa05af11c..5c0e592f4caeda1 100644 --- a/Doc/deprecations/pending-removal-in-3.15.rst +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -3,14 +3,9 @@ Pending removal in Python 3.15 * The import system: - * Setting :attr:`~module.__cached__` on a module while + * Setting ``__cached__`` on a module while failing to set :attr:`__spec__.cached ` - is deprecated. In Python 3.15, :attr:`!__cached__` will cease to be set or - take into consideration by the import system or standard library. (:gh:`97879`) - - * Setting :attr:`~module.__package__` on a module while - failing to set :attr:`__spec__.parent ` - is deprecated. In Python 3.15, :attr:`!__package__` will cease to be set or + is deprecated. In Python 3.15, ``__cached__`` will cease to be set or take into consideration by the import system or standard library. (:gh:`97879`) * :mod:`ctypes`: @@ -33,16 +28,6 @@ Pending removal in Python 3.15 * ``load_module()`` method: use ``exec_module()`` instead. -* :class:`locale`: - - * The :func:`~locale.getdefaultlocale` function - has been deprecated since Python 3.11. - Its removal was originally planned for Python 3.13 (:gh:`90817`), - but has been postponed to Python 3.15. - Use :func:`~locale.getlocale`, :func:`~locale.setlocale`, - and :func:`~locale.getencoding` instead. - (Contributed by Hugo van Kemenade in :gh:`111187`.) - * :mod:`pathlib`: * :meth:`!.PurePath.is_reserved` @@ -64,13 +49,13 @@ Pending removal in Python 3.15 * :func:`~threading.RLock` will take no arguments in Python 3.15. Passing any arguments has been deprecated since Python 3.14, - as the Python version does not permit any arguments, + as the Python version does not permit any arguments, but the C version allows any number of positional or keyword arguments, ignoring every argument. * :mod:`types`: - * :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was + * :class:`types.CodeType`: Accessing :attr:`!codeobject.co_lnotab` was deprecated in :pep:`626` since 3.10 and was planned to be removed in 3.12, but it only got a proper :exc:`DeprecationWarning` in 3.12. @@ -92,7 +77,7 @@ Pending removal in Python 3.15 Use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})`` to create a TypedDict with zero field. - * The :func:`typing.no_type_check_decorator` decorator function + * The :deco:`!typing.no_type_check_decorator` decorator function has been deprecated since Python 3.13. After eight years in the :mod:`typing` module, it has yet to be supported by any major type checker. @@ -107,6 +92,6 @@ Pending removal in Python 3.15 * :mod:`zipimport`: - * :meth:`~zipimport.zipimporter.load_module` has been deprecated since + * :meth:`!zipimport.zipimporter.load_module` has been deprecated since Python 3.10. Use :meth:`~zipimport.zipimporter.exec_module` instead. - (Contributed by Jiahao Li in :gh:`125746`.) + (:gh:`125746`.) diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index cdd76ee693f31bb..5a28cc766a95961 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -8,6 +8,13 @@ Pending removal in Python 3.16 is deprecated. In Python 3.16, :attr:`!__loader__` will cease to be set or taken into consideration by the import system or the standard library. + * Setting :attr:`~module.__package__` on a module while + failing to set :attr:`__spec__.parent ` + is deprecated. In Python 3.16, :attr:`!__package__` will cease to be + taken into consideration by the import system or standard library. (:gh:`97879`) + +* The bundled copy of ``libmpdec``. + * :mod:`array`: * The ``'u'`` format code (:c:type:`wchar_t`) @@ -63,9 +70,9 @@ Pending removal in Python 3.16 * :mod:`logging`: - Support for custom logging handlers with the *strm* argument is deprecated - and scheduled for removal in Python 3.16. Define handlers with the *stream* - argument instead. (Contributed by Mariusz Felisiak in :gh:`115032`.) + * Support for custom logging handlers with the *strm* argument is deprecated + and scheduled for removal in Python 3.16. Define handlers with the *stream* + argument instead. (Contributed by Mariusz Felisiak in :gh:`115032`.) * :mod:`mimetypes`: @@ -84,12 +91,12 @@ Pending removal in Python 3.16 * :mod:`symtable`: - * The :meth:`Class.get_methods ` method + * The :meth:`!symtable.Class.get_methods` method has been deprecated since Python 3.14. * :mod:`sys`: - * The :func:`~sys._enablelegacywindowsfsencoding` function + * The :func:`!_enablelegacywindowsfsencoding` function has been deprecated since Python 3.13. Use the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable instead. @@ -101,5 +108,5 @@ Pending removal in Python 3.16 * :mod:`tarfile`: - * The undocumented and unused :attr:`!TarFile.tarfile` attribute + * The undocumented and unused :attr:`!TarInfo.tarfile` attribute has been deprecated since Python 3.13. diff --git a/Doc/deprecations/pending-removal-in-3.17.rst b/Doc/deprecations/pending-removal-in-3.17.rst index 370b98307e52280..8ee7f335cc95148 100644 --- a/Doc/deprecations/pending-removal-in-3.17.rst +++ b/Doc/deprecations/pending-removal-in-3.17.rst @@ -1,6 +1,47 @@ Pending removal in Python 3.17 ------------------------------ +* :mod:`datetime`: + + * :meth:`~datetime.datetime.strptime` calls using a format string containing + ``%e`` (day of month) without a year. + This has been deprecated since Python 3.15. + (Contributed by Stan Ulbrych in :gh:`70647`.) + + +* :mod:`collections.abc`: + + - :class:`collections.abc.ByteString` is scheduled for removal in Python 3.17. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the :ref:`buffer protocol ` at runtime. For use + in type annotations, either use :class:`~collections.abc.Buffer` or a union + that explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was an + instance of :class:`!ByteString` never actually told you anything useful + about the object. Other common buffer types such as :class:`memoryview` + were also never understood as subtypes of :class:`!ByteString` (either at + runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. + (Contributed by Shantanu Jain in :gh:`91896`.) + + +* :mod:`encodings`: + + - Passing non-ascii *encoding* names to :func:`encodings.normalize_encoding` + is deprecated and scheduled for removal in Python 3.17. + (Contributed by Stan Ulbrych in :gh:`136702`.) + +* :mod:`webbrowser`: + + - :class:`!webbrowser.MacOSXOSAScript` is deprecated in favour of + :class:`!webbrowser.MacOS`. (:gh:`137586`) + * :mod:`typing`: - Before Python 3.14, old-style unions were implemented using the private class @@ -8,3 +49,22 @@ Pending removal in Python 3.17 but it has been retained for backward compatibility, with removal scheduled for Python 3.17. Users should use documented introspection helpers like :func:`typing.get_origin` and :func:`typing.get_args` instead of relying on private implementation details. + - :class:`typing.ByteString`, deprecated since Python 3.9, is scheduled for removal in + Python 3.17. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the :ref:`buffer protocol ` at runtime. For use + in type annotations, either use :class:`~collections.abc.Buffer` or a union + that explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was an + instance of :class:`!ByteString` never actually told you anything useful + about the object. Other common buffer types such as :class:`memoryview` + were also never understood as subtypes of :class:`!ByteString` (either at + runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. + (Contributed by Shantanu Jain in :gh:`91896`.) diff --git a/Doc/deprecations/pending-removal-in-3.18.rst b/Doc/deprecations/pending-removal-in-3.18.rst new file mode 100644 index 000000000000000..19113aab981bbc6 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.18.rst @@ -0,0 +1,18 @@ +Pending removal in Python 3.18 +------------------------------ + +* No longer accept a boolean value when a file descriptor is expected. + (Contributed by Serhiy Storchaka in :gh:`82626`.) + +* :mod:`decimal`: + + * The non-standard and undocumented :class:`~decimal.Decimal` format + specifier ``'N'``, which is only supported in the :mod:`!decimal` module's + C implementation, has been deprecated since Python 3.13. + (Contributed by Serhiy Storchaka in :gh:`89902`.) + +* Deprecations defined by :pep:`829`: + + * ``import`` lines in :file:`{name}.pth` files are silently ignored. + + (Contributed by Barry Warsaw in :gh:`148641`.) diff --git a/Doc/deprecations/pending-removal-in-3.19.rst b/Doc/deprecations/pending-removal-in-3.19.rst index 25f9cba390de68e..4a58c606ab7596e 100644 --- a/Doc/deprecations/pending-removal-in-3.19.rst +++ b/Doc/deprecations/pending-removal-in-3.19.rst @@ -22,3 +22,21 @@ Pending removal in Python 3.19 supported depending on the backend implementation of hash functions. Prefer passing the initial data as a positional argument for maximum backwards compatibility. + +* :mod:`http.cookies`: + + * :meth:`http.cookies.Morsel.js_output` is deprecated and will be + removed in Python 3.19. + + * :meth:`http.cookies.BaseCookie.js_output` is deprecated and will be + removed in Python 3.19. + +* :mod:`imaplib`: + + * Altering :attr:`IMAP4.file ` is now deprecated + and slated for removal in Python 3.19. This property is now unused + and changing its value does not automatically close the current file. + + Before Python 3.14, this property was used to implement the corresponding + ``read()`` and ``readline()`` methods for :class:`~imaplib.IMAP4` but this + is no longer the case since then. diff --git a/Doc/deprecations/pending-removal-in-3.20.rst b/Doc/deprecations/pending-removal-in-3.20.rst new file mode 100644 index 000000000000000..011565dfbb090d4 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.20.rst @@ -0,0 +1,55 @@ +Pending removal in Python 3.20 +------------------------------ + +* Calling the ``__new__()`` method of :class:`struct.Struct` without the + *format* argument is deprecated and will be removed in Python 3.20. Calling + :meth:`~object.__init__` method on initialized :class:`~struct.Struct` + objects is deprecated and will be removed in Python 3.20. + + (Contributed by Sergey B Kirpichev and Serhiy Storchaka in :gh:`143715`.) + +* The ``__version__``, ``version`` and ``VERSION`` attributes have been + deprecated in these standard library modules and will be removed in + Python 3.20. Use :py:data:`sys.version_info` instead. + + - :mod:`argparse` + - :mod:`csv` + - :mod:`ctypes` + - :mod:`!ctypes.macholib` + - :mod:`decimal` (use :data:`decimal.SPEC_VERSION` instead) + - :mod:`http.server` + - :mod:`imaplib` + - :mod:`ipaddress` + - :mod:`json` + - :mod:`logging` (``__date__`` also deprecated) + - :mod:`optparse` + - :mod:`pickle` + - :mod:`platform` + - :mod:`re` + - :mod:`socketserver` + - :mod:`tabnanny` + - :mod:`tarfile` + - :mod:`tkinter.font` + - :mod:`tkinter.ttk` + - :mod:`wsgiref.simple_server` + - :mod:`xml.etree.ElementTree` + - :mod:`!xml.sax.expatreader` + - :mod:`xml.sax.handler` + - :mod:`zlib` + + (Contributed by Hugo van Kemenade and Stan Ulbrych in :gh:`76007`.) + +* Deprecations defined by :pep:`829`: + + * Warnings are produced for ``import`` lines found in :file:`{name}.pth` + files. + + * :file:`{name}.pth` files are no longer decoded in the locale encoding by + default. They **MUST** be encoded in ``utf-8-sig``. + + (Contributed by Barry Warsaw in :gh:`148641`.) + +* :mod:`ast`: + + * Creating instances of abstract AST nodes (such as :class:`ast.AST` + or :class:`!ast.expr`) is deprecated and will raise an error in Python 3.20. diff --git a/Doc/deprecations/pending-removal-in-3.21.rst b/Doc/deprecations/pending-removal-in-3.21.rst new file mode 100644 index 000000000000000..18b89a20e4a2082 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.21.rst @@ -0,0 +1,19 @@ +Pending removal in Python 3.21 +------------------------------ + +* :mod:`abc` + + * Soft-deprecated since Python 3.3 :class:`abc.abstractclassmethod`, + :class:`abc.abstractstaticmethod`, and :class:`abc.abstractproperty` + now raise a :exc:`DeprecationWarning`. + These classes will be removed in Python 3.21, instead + use :func:`abc.abstractmethod` with :func:`classmethod`, + :func:`staticmethod`, and :class:`property` respectively. + +* :mod:`ast`: + + * Classes ``slice``, ``Index``, ``ExtSlice``, ``Suite``, ``Param``, + ``AugLoad`` and ``AugStore``, will be removed in Python 3.21. These types + are not generated by the parser or accepted by the code generator. + * The ``dims`` property of ``ast.Tuple`` will be removed in Python 3.21. Use + the ``ast.Tuple.elts`` property instead. diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst index edb672ed8ad9a29..74f98d33a4b61f1 100644 --- a/Doc/deprecations/pending-removal-in-future.rst +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -15,7 +15,6 @@ although there is currently no date scheduled for their removal. * :mod:`builtins`: - * ``bool(NotImplemented)``. * Generators: ``throw(type, exc, tb)`` and ``athrow(type, exc, tb)`` signature is deprecated: use ``throw(exc)`` and ``athrow(exc)`` instead, the single argument signature. @@ -36,7 +35,6 @@ although there is currently no date scheduled for their removal. * Support for ``__complex__()`` method returning a strict subclass of :class:`complex`: these methods will be required to return an instance of :class:`complex`. - * Delegation of ``int()`` to ``__trunc__()`` method. * Passing a complex number as the *real* or *imag* argument in the :func:`complex` constructor is now deprecated; it should only be passed as a single positional argument. @@ -49,7 +47,7 @@ although there is currently no date scheduled for their removal. * :mod:`codecs`: use :func:`open` instead of :func:`codecs.open`. (:gh:`133038`) -* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method +* :attr:`!codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method instead. * :mod:`datetime`: @@ -77,7 +75,15 @@ although there is currently no date scheduled for their removal. * :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use BytesIO and binary mode instead. -* :mod:`os`: Calling :func:`os.register_at_fork` in multi-threaded process. +* :mod:`os`: Calling :func:`os.register_at_fork` in a multi-threaded process. + +* :mod:`os.path`: :func:`os.path.commonprefix` is deprecated, use + :func:`os.path.commonpath` for path prefixes. The :func:`os.path.commonprefix` + function is being deprecated due to having a misleading name and module. + The function is not safe to use for path prefixes despite being included in a + module about path manipulation, meaning it is easy to accidentally + introduce path traversal vulnerabilities into Python programs by using this + function. * :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is deprecated, use an exception instance. diff --git a/Doc/deprecations/soft-deprecations.rst b/Doc/deprecations/soft-deprecations.rst new file mode 100644 index 000000000000000..a270052788ef2a1 --- /dev/null +++ b/Doc/deprecations/soft-deprecations.rst @@ -0,0 +1,21 @@ +Soft deprecations +----------------- + +There are no plans to remove :term:`soft deprecated` APIs. + +* :func:`re.match` and :meth:`re.Pattern.match` are now + :term:`soft deprecated` in favor of the new :func:`re.prefixmatch` and + :meth:`re.Pattern.prefixmatch` APIs, which have been added as alternate, + more explicit names. These are intended to be used to alleviate confusion + around what *match* means by following the Zen of Python's *"Explicit is + better than implicit"* mantra. Most other language regular expression + libraries use an API named *match* to mean what Python has always called + *search*. + + We **do not** plan to remove the older :func:`!match` name, as it has been + used in code for over 30 years. Code supporting older versions of Python + should continue to use :func:`!match`, while new code should prefer + :func:`!prefixmatch`. See :ref:`prefixmatch-vs-match`. + + (Contributed by Gregory P. Smith in :gh:`86519` and + Hugo van Kemenade in :gh:`148100`.) diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index a89a69043c0f9f9..d33cbd2813d637b 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -3,129 +3,20 @@ .. _extending-intro: -****************************** -Extending Python with C or C++ -****************************** +******************************** +Using the C API: Assorted topics +******************************** -It is quite easy to add new built-in modules to Python, if you know how to -program in C. Such :dfn:`extension modules` can do two things that can't be -done directly in Python: they can implement new built-in object types, and they -can call C library functions and system calls. - -To support extensions, the Python API (Application Programmers Interface) -defines a set of functions, macros and variables that provide access to most -aspects of the Python run-time system. The Python API is incorporated in a C -source file by including the header ``"Python.h"``. - -The compilation of an extension module depends on its intended use as well as on -your system setup; details are given in later chapters. - -.. note:: - - The C extension interface is specific to CPython, and extension modules do - not work on other Python implementations. In many cases, it is possible to - avoid writing C extensions and preserve portability to other implementations. - For example, if your use case is calling C library functions or system calls, - you should consider using the :mod:`ctypes` module or the `cffi - `_ library rather than writing - custom C code. - These modules let you write Python code to interface with C code and are more - portable between implementations of Python than writing and compiling a C - extension module. - - -.. _extending-simpleexample: - -A Simple Example -================ - -Let's create an extension module called ``spam`` (the favorite food of Monty -Python fans...) and let's say we want to create a Python interface to the C -library function :c:func:`system` [#]_. This function takes a null-terminated -character string as argument and returns an integer. We want this function to -be callable from Python as follows: - -.. code-block:: pycon - - >>> import spam - >>> status = spam.system("ls -l") - -Begin by creating a file :file:`spammodule.c`. (Historically, if a module is -called ``spam``, the C file containing its implementation is called -:file:`spammodule.c`; if the module name is very long, like ``spammify``, the -module name can be just :file:`spammify.c`.) - -The first two lines of our file can be:: - - #define PY_SSIZE_T_CLEAN - #include - -which pulls in the Python API (you can add a comment describing the purpose of -the module and a copyright notice if you like). - -.. note:: - - Since Python may define some pre-processor definitions which affect the standard - headers on some systems, you *must* include :file:`Python.h` before any standard - headers are included. - - ``#define PY_SSIZE_T_CLEAN`` was used to indicate that ``Py_ssize_t`` should be - used in some APIs instead of ``int``. - It is not necessary since Python 3.13, but we keep it here for backward compatibility. - See :ref:`arg-parsing-string-and-buffers` for a description of this macro. - -All user-visible symbols defined by :file:`Python.h` have a prefix of ``Py`` or -``PY``, except those defined in standard header files. For convenience, and -since they are used extensively by the Python interpreter, ``"Python.h"`` -includes a few standard header files: ````, ````, -````, and ````. If the latter header file does not exist on -your system, it declares the functions :c:func:`malloc`, :c:func:`free` and -:c:func:`realloc` directly. - -The next thing we add to our module file is the C function that will be called -when the Python expression ``spam.system(string)`` is evaluated (we'll see -shortly how it ends up being called):: - - static PyObject * - spam_system(PyObject *self, PyObject *args) - { - const char *command; - int sts; - - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - sts = system(command); - return PyLong_FromLong(sts); - } - -There is a straightforward translation from the argument list in Python (for -example, the single expression ``"ls -l"``) to the arguments passed to the C -function. The C function always has two arguments, conventionally named *self* -and *args*. - -The *self* argument points to the module object for module-level functions; -for a method it would point to the object instance. - -The *args* argument will be a pointer to a Python tuple object containing the -arguments. Each item of the tuple corresponds to an argument in the call's -argument list. The arguments are Python objects --- in order to do anything -with them in our C function we have to convert them to C values. The function -:c:func:`PyArg_ParseTuple` in the Python API checks the argument types and -converts them to C values. It uses a template string to determine the required -types of the arguments as well as the types of the C variables into which to -store the converted values. More about this later. - -:c:func:`PyArg_ParseTuple` returns true (nonzero) if all arguments have the right -type and its components have been stored in the variables whose addresses are -passed. It returns false (zero) if an invalid argument list was passed. In the -latter case it also raises an appropriate exception so the calling function can -return ``NULL`` immediately (as we saw in the example). +The :ref:`tutorial ` walked you through +creating a C API extension module, but left many areas unexplained. +This document looks at several concepts that you'll need to learn +in order to write more complex extensions. .. _extending-errors: -Intermezzo: Errors and Exceptions -================================= +Errors and Exceptions +===================== An important convention throughout the Python interpreter is the following: when a function fails, it should set an exception condition and return an error value @@ -296,194 +187,14 @@ call to :c:func:`PyErr_SetString` as shown below:: } -.. _backtoexample: - -Back to the Example -=================== - -Going back to our example function, you should now be able to understand this -statement:: - - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - -It returns ``NULL`` (the error indicator for functions returning object pointers) -if an error is detected in the argument list, relying on the exception set by -:c:func:`PyArg_ParseTuple`. Otherwise the string value of the argument has been -copied to the local variable :c:data:`!command`. This is a pointer assignment and -you are not supposed to modify the string to which it points (so in Standard C, -the variable :c:data:`!command` should properly be declared as ``const char -*command``). - -The next statement is a call to the Unix function :c:func:`system`, passing it -the string we just got from :c:func:`PyArg_ParseTuple`:: - - sts = system(command); - -Our :func:`!spam.system` function must return the value of :c:data:`!sts` as a -Python object. This is done using the function :c:func:`PyLong_FromLong`. :: - - return PyLong_FromLong(sts); - -In this case, it will return an integer object. (Yes, even integers are objects -on the heap in Python!) - -If you have a C function that returns no useful argument (a function returning -:c:expr:`void`), the corresponding Python function must return ``None``. You -need this idiom to do so (which is implemented by the :c:macro:`Py_RETURN_NONE` -macro):: - - Py_INCREF(Py_None); - return Py_None; - -:c:data:`Py_None` is the C name for the special Python object ``None``. It is a -genuine Python object rather than a ``NULL`` pointer, which means "error" in most -contexts, as we have seen. - - -.. _methodtable: - -The Module's Method Table and Initialization Function -===================================================== - -I promised to show how :c:func:`!spam_system` is called from Python programs. -First, we need to list its name and address in a "method table":: - - static PyMethodDef spam_methods[] = { - ... - {"system", spam_system, METH_VARARGS, - "Execute a shell command."}, - ... - {NULL, NULL, 0, NULL} /* Sentinel */ - }; - -Note the third entry (``METH_VARARGS``). This is a flag telling the interpreter -the calling convention to be used for the C function. It should normally always -be ``METH_VARARGS`` or ``METH_VARARGS | METH_KEYWORDS``; a value of ``0`` means -that an obsolete variant of :c:func:`PyArg_ParseTuple` is used. - -When using only ``METH_VARARGS``, the function should expect the Python-level -parameters to be passed in as a tuple acceptable for parsing via -:c:func:`PyArg_ParseTuple`; more information on this function is provided below. - -The :c:macro:`METH_KEYWORDS` bit may be set in the third field if keyword -arguments should be passed to the function. In this case, the C function should -accept a third ``PyObject *`` parameter which will be a dictionary of keywords. -Use :c:func:`PyArg_ParseTupleAndKeywords` to parse the arguments to such a -function. - -The method table must be referenced in the module definition structure:: - - static struct PyModuleDef spam_module = { - ... - .m_methods = spam_methods, - ... - }; - -This structure, in turn, must be passed to the interpreter in the module's -initialization function. The initialization function must be named -:c:func:`!PyInit_name`, where *name* is the name of the module, and should be the -only non-\ ``static`` item defined in the module file:: - - PyMODINIT_FUNC - PyInit_spam(void) - { - return PyModuleDef_Init(&spam_module); - } - -Note that :c:macro:`PyMODINIT_FUNC` declares the function as ``PyObject *`` return type, -declares any special linkage declarations required by the platform, and for C++ -declares the function as ``extern "C"``. - -:c:func:`!PyInit_spam` is called when each interpreter imports its module -:mod:`!spam` for the first time. (See below for comments about embedding Python.) -A pointer to the module definition must be returned via :c:func:`PyModuleDef_Init`, -so that the import machinery can create the module and store it in ``sys.modules``. - -When embedding Python, the :c:func:`!PyInit_spam` function is not called -automatically unless there's an entry in the :c:data:`PyImport_Inittab` table. -To add the module to the initialization table, use :c:func:`PyImport_AppendInittab`, -optionally followed by an import of the module:: - - #define PY_SSIZE_T_CLEAN - #include - - int - main(int argc, char *argv[]) - { - PyStatus status; - PyConfig config; - PyConfig_InitPythonConfig(&config); - - /* Add a built-in module, before Py_Initialize */ - if (PyImport_AppendInittab("spam", PyInit_spam) == -1) { - fprintf(stderr, "Error: could not extend in-built modules table\n"); - exit(1); - } - - /* Pass argv[0] to the Python interpreter */ - status = PyConfig_SetBytesString(&config, &config.program_name, argv[0]); - if (PyStatus_Exception(status)) { - goto exception; - } - - /* Initialize the Python interpreter. Required. - If this step fails, it will be a fatal error. */ - status = Py_InitializeFromConfig(&config); - if (PyStatus_Exception(status)) { - goto exception; - } - PyConfig_Clear(&config); - - /* Optionally import the module; alternatively, - import can be deferred until the embedded script - imports it. */ - PyObject *pmodule = PyImport_ImportModule("spam"); - if (!pmodule) { - PyErr_Print(); - fprintf(stderr, "Error: could not import module 'spam'\n"); - } - - // ... use Python C API here ... - - return 0; - - exception: - PyConfig_Clear(&config); - Py_ExitStatusException(status); - } - -.. note:: - - If you declare a global variable or a local static one, the module may - experience unintended side-effects on re-initialisation, for example when - removing entries from ``sys.modules`` or importing compiled modules into - multiple interpreters within a process - (or following a :c:func:`fork` without an intervening :c:func:`exec`). - If module state is not yet fully :ref:`isolated `, - authors should consider marking the module as having no support for subinterpreters - (via :c:macro:`Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED`). - -A more substantial example module is included in the Python source distribution -as :file:`Modules/xxlimited.c`. This file may be used as a template or simply -read as an example. - - .. _compilation: -Compilation and Linkage -======================= - -There are two more things to do before you can use your new extension: compiling -and linking it with the Python system. If you use dynamic loading, the details -may depend on the style of dynamic loading your system uses; see the chapters -about building extension modules (chapter :ref:`building`) and additional -information that pertains only to building on Windows (chapter -:ref:`building-on-windows`) for more information about this. +Embedding an extension +====================== -If you can't use dynamic loading, or if you want to make your module a permanent +If you want to make your module a permanent part of the Python interpreter, you will have to change the configuration setup -and rebuild the interpreter. Luckily, this is very simple on Unix: just place +and rebuild the interpreter. On Unix, place your file (:file:`spammodule.c` for example) in the :file:`Modules/` directory of an unpacked source distribution, add a line to the file :file:`Modules/Setup.local` describing your file: @@ -511,7 +222,7 @@ on the line in the configuration file as well, for instance: Calling Python Functions from C =============================== -So far we have concentrated on making C functions callable from Python. The +The tutorial concentrated on making C functions callable from Python. The reverse is also useful: calling Python functions from C. This is especially the case for libraries that support so-called "callback" functions. If a C interface makes use of callbacks, the equivalent Python often needs to provide a @@ -556,7 +267,7 @@ be part of a module definition:: } This function must be registered with the interpreter using the -:c:macro:`METH_VARARGS` flag; this is described in section :ref:`methodtable`. The +:c:macro:`METH_VARARGS` flag in :c:type:`PyMethodDef.ml_flags`. The :c:func:`PyArg_ParseTuple` function and its arguments are documented in section :ref:`parsetuple`. @@ -651,14 +362,21 @@ the above example, we use :c:func:`Py_BuildValue` to construct the dictionary. : Py_DECREF(result); +.. index:: single: PyArg_ParseTuple (C function) + .. _parsetuple: Extracting Parameters in Extension Functions ============================================ -.. index:: single: PyArg_ParseTuple (C function) +The :ref:`tutorial ` uses a ":c:data:`METH_O`" +function, which is limited to a single Python argument. +If you want more, you can use :c:data:`METH_VARARGS` instead. +With this flag, the C function will receive a :py:class:`tuple` of arguments +instead of a single object. -The :c:func:`PyArg_ParseTuple` function is declared as follows:: +For unpacking the tuple, CPython provides the :c:func:`PyArg_ParseTuple` +function, declared as follows:: int PyArg_ParseTuple(PyObject *arg, const char *format, ...); @@ -668,6 +386,19 @@ whose syntax is explained in :ref:`arg-parsing` in the Python/C API Reference Manual. The remaining arguments must be addresses of variables whose type is determined by the format string. +For example, to receive a single Python :py:class:`str` object and turn it +into a C buffer, you would use ``"s"`` as the format string:: + + const char *command; + if (!PyArg_ParseTuple(args, "s", &command)) { + return NULL; + } + +If an error is detected in the argument list, :c:func:`!PyArg_ParseTuple` +returns ``NULL`` (the error indicator for functions returning object pointers); +your function may return ``NULL``, relying on the exception set by +:c:func:`PyArg_ParseTuple`. + Note that while :c:func:`PyArg_ParseTuple` checks that the Python arguments have the required types, it cannot check the validity of the addresses of C variables passed to the call: if you make mistakes there, your code will probably crash or @@ -678,7 +409,6 @@ Note that any Python object references which are provided to the caller are Some example calls:: - #define PY_SSIZE_T_CLEAN #include :: @@ -748,6 +478,17 @@ Some example calls:: Keyword Parameters for Extension Functions ========================================== +If you also want your function to accept +:term:`keyword arguments `, use the :c:data:`METH_KEYWORDS` +flag in combination with :c:data:`METH_VARARGS`. +(:c:data:`!METH_KEYWORDS` can also be used with other flags; see its +documentation for the allowed combinations.) + +In this case, the C function should accept a third ``PyObject *`` parameter +which will be a dictionary of keywords. +Use :c:func:`PyArg_ParseTupleAndKeywords` to parse the arguments to such a +function. + .. index:: single: PyArg_ParseTupleAndKeywords (C function) The :c:func:`PyArg_ParseTupleAndKeywords` function is declared as follows:: @@ -808,19 +549,6 @@ Philbrick (philbrick@hks.com):: {NULL, NULL, 0, NULL} /* sentinel */ }; - static struct PyModuleDef keywdarg_module = { - .m_base = PyModuleDef_HEAD_INIT, - .m_name = "keywdarg", - .m_size = 0, - .m_methods = keywdarg_methods, - }; - - PyMODINIT_FUNC - PyInit_keywdarg(void) - { - return PyModuleDef_Init(&keywdarg_module); - } - .. _buildvalue: @@ -961,11 +689,11 @@ needed. Ownership of a reference can be transferred. There are three ways to dispose of an owned reference: pass it on, store it, or call :c:func:`Py_DECREF`. Forgetting to dispose of an owned reference creates a memory leak. -It is also possible to :dfn:`borrow` [#]_ a reference to an object. The +It is also possible to :dfn:`borrow` [#borrow]_ a reference to an object. The borrower of a reference should not call :c:func:`Py_DECREF`. The borrower must not hold on to the object longer than the owner from which it was borrowed. Using a borrowed reference after the owner has disposed of it risks using freed -memory and should be avoided completely [#]_. +memory and should be avoided completely [#dont-check-refcount]_. The advantage of borrowing over owning a reference is that you don't need to take care of disposing of the reference on all possible paths through the code @@ -1059,7 +787,14 @@ references to all its items, so when item 1 is replaced, it has to dispose of the original item 1. Now let's suppose the original item 1 was an instance of a user-defined class, and let's further suppose that the class defined a :meth:`!__del__` method. If this class instance has a reference count of 1, -disposing of it will call its :meth:`!__del__` method. +disposing of it will call its :meth:`!__del__` method. Internally, +:c:func:`PyList_SetItem` calls :c:func:`Py_DECREF` on the replaced item, +which invokes replaced item's corresponding +:c:member:`~PyTypeObject.tp_dealloc` function. During +deallocation, :c:member:`~PyTypeObject.tp_dealloc` calls +:c:member:`~PyTypeObject.tp_finalize`, which is mapped to the +:meth:`!__del__` method for class instances (see :pep:`442`). This entire +sequence happens synchronously within the :c:func:`PyList_SetItem` call. Since it is written in Python, the :meth:`!__del__` method can execute arbitrary Python code. Could it perhaps do something to invalidate the reference to @@ -1137,7 +872,7 @@ checking. The C function calling mechanism guarantees that the argument list passed to C functions (``args`` in the examples) is never ``NULL`` --- in fact it guarantees -that it is always a tuple [#]_. +that it is always a tuple [#old-calling-convention]_. It is a severe error to ever let a ``NULL`` pointer "escape" to the Python user. @@ -1168,9 +903,6 @@ define this symbol). Providing a C API for an Extension Module ========================================= -.. sectionauthor:: Konrad Hinsen - - Many extension modules just provide new functions and types to be used from Python, but sometimes the code in an extension module can be useful for other extension modules. For example, an extension module could implement a type @@ -1194,8 +926,8 @@ the module whose functions one wishes to call might not have been loaded yet! Portability therefore requires not to make any assumptions about symbol visibility. This means that all symbols in extension modules should be declared ``static``, except for the module's initialization function, in order to -avoid name clashes with other extension modules (as discussed in section -:ref:`methodtable`). And it means that symbols that *should* be accessible from +avoid name clashes with other extension modules. And it means that symbols +that *should* be accessible from other extension modules must be exported in a different way. Python provides a special mechanism to pass C-level information (pointers) from @@ -1237,8 +969,9 @@ file corresponding to the module provides a macro that takes care of importing the module and retrieving its C API pointers; client modules only have to call this macro before accessing the C API. -The exporting module is a modification of the :mod:`!spam` module from section -:ref:`extending-simpleexample`. The function :func:`!spam.system` does not call +The exporting module is a modification of the :mod:`!spam` module from the +:ref:`tutorial `. +The function :func:`!spam.system` does not call the C library function :c:func:`system` directly, but a function :c:func:`!PySpam_System`, which would of course do something more complicated in reality (such as adding "spam" to every command). This function @@ -1380,15 +1113,14 @@ code distribution). .. rubric:: Footnotes -.. [#] An interface for this function already exists in the standard module :mod:`os` - --- it was chosen as a simple and straightforward example. - -.. [#] The metaphor of "borrowing" a reference is not completely correct: the owner - still has a copy of the reference. +.. [#borrow] The metaphor of "borrowing" a reference is not completely correct: + the owner still has a copy of the reference. -.. [#] Checking that the reference count is at least 1 **does not work** --- the +.. [#dont-check-refcount] Checking that the reference count is at least 1 + **does not work** --- the reference count itself could be in freed memory and may thus be reused for another object! -.. [#] These guarantees don't hold when you use the "old" style calling convention --- +.. [#old-calling-convention] These guarantees don't hold when you use the + "old" style calling convention --- this is still found in much existing code. diff --git a/Doc/extending/first-extension-module.rst b/Doc/extending/first-extension-module.rst new file mode 100644 index 000000000000000..55a772e2aca24f5 --- /dev/null +++ b/Doc/extending/first-extension-module.rst @@ -0,0 +1,699 @@ +.. highlight:: c + + +.. _extending-simpleexample: +.. _first-extension-module: + +********************************* +Your first C API extension module +********************************* + +This tutorial will take you through creating a simple +Python extension module written in C or C++. + +We will use the low-level Python C API directly. +For easier ways to create extension modules, see +the :ref:`recommended third party tools `. + +The tutorial assumes basic knowledge about Python: you should be able to +define functions in Python code before starting to write them in C. +See :ref:`tutorial-index` for an introduction to Python itself. + +The tutorial should be approachable for anyone who can write a basic C library. +While we will mention several concepts that a C beginner would not be expected +to know, like ``static`` functions or linkage declarations, understanding these +is not necessary for success. + +We will focus on giving you a "feel" of what Python's C API is like. +It will not teach you important concepts, like error handling +and reference counting, which are covered in later chapters. + +We will assume that you use a Unix-like system (including macOS and +Linux), or Windows. +On other systems, you might need to adjust some details -- for example, +a system command name. + +You need to have a suitable C compiler and Python development headers installed. +On Linux, headers are often in a package like ``python3-dev`` +or ``python3-devel``. + +You need to be able to install Python packages. +This tutorial uses `pip `__ (``pip install``), but you +can substitute any tool that can build and install ``pyproject.toml``-based +projects, like `uv `_ (``uv pip install``). +Preferably, have a :ref:`virtual environment ` activated. + + +.. note:: + + This tutorial uses APIs that were added in CPython 3.15. + To create an extension that's compatible with earlier versions of CPython, + please follow an earlier version of this documentation. + + This tutorial uses C syntax added in C11 and C++20. + If your extension needs to be compatible with earlier standards, + please follow tutorials in documentation for Python 3.14 or below. + + +What we'll do +============= + +Let's create an extension module called ``spam`` [#why-spam]_, +which will include a Python interface to the C +standard library function :c:func:`system`. +This function is defined in ``stdlib.h``. +It takes a C string as argument, runs the argument as a system +command, and returns a result value as an integer. +A manual page for :c:func:`system` might summarize it this way:: + + #include + int system(const char *command); + +Note that like many functions in the C standard library, +this function is already exposed in Python. +In production, use :py:func:`os.system` or :py:func:`subprocess.run` +rather than the module you'll write here. + +We want this function to be callable from Python as follows: + +.. code-block:: pycon + + >>> import spam + >>> status = spam.system("whoami") + User Name + >>> status + 0 + +.. note:: + + The system command ``whoami`` prints out your username. + It's useful in tutorials like this one because it has the same name on + both Unix and Windows. + + +Start with the headers +====================== + +Begin by creating a directory for this tutorial, and switching to it +on the command line. +Then, create a file named :file:`spammodule.c` in your directory. +[#why-spammodule]_ + +In this file, we'll include two headers: :file:`Python.h` to pull in +all declarations of the Python C API, and :file:`stdlib.h` for the +:c:func:`system` function. [#stdlib-h]_ + +Add the following lines to :file:`spammodule.c`: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-at: + :end-at: + +Be sure to put :file:`stdlib.h`, and any other standard library includes, +*after* :file:`Python.h`. +On some systems, Python may define some pre-processor definitions +that affect the standard headers. + + +Running your build tool +======================= + +With only the includes in place, your extension won't do anything. +Still, it's a good time to compile it and try to import it. +This will ensure that your build tool works, so that you can make +and test incremental changes as you follow the rest of the text. + +CPython itself does not come with a tool to build extension modules; +it is recommended to use a third-party project for this. +In this tutorial, we'll use `meson-python`_. +(If you want to use another one, see :ref:`first-extension-other-tools`.) + +.. at the time of writing, meson-python has the least overhead for a + simple extension using PyModExport. + Change this if another tool makes things easier. + +``meson-python`` requires defining a "project" using two extra files. + +First, add ``pyproject.toml`` with these contents: + +.. code-block:: toml + + [build-system] + build-backend = 'mesonpy' + requires = ['meson-python'] + + [project] + # Placeholder project information + # (change this before distributing the module) + name = 'sampleproject' + version = '0' + +Then, create ``meson.build`` containing the following: + +.. code-block:: meson + + project('sampleproject', 'c') + + py = import('python').find_installation(pure: false) + + py.extension_module( + 'spam', # name of the importable Python module + 'spammodule.c', # the C source file + install: true, + ) + +.. note:: + + See the `meson-python documentation `_ for details on + configuration. + +Now, build install the *project in the current directory* (``.``) via ``pip``: + +.. code-block:: sh + + python -m pip -v install . + +The ``-v`` (``--verbose``) option causes ``pip`` to show the output from +the compiler, which is often useful during development. + +.. tip:: + + If you don't have ``pip`` installed, run ``python -m ensurepip``, + preferably in a :ref:`virtual environment `. + (Or, if you prefer another tool that can build and install + ``pyproject.toml``-based projects, use that.) + +.. _meson-python: https://mesonbuild.com/meson-python/ +.. _virtual environment: https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments + +Note that you will need to run this command again every time you change your +extension. +Unlike Python, C has an explicit compilation step. + +When your extension is compiled and installed, start Python and try to +import it. +This should fail with the following exception: + +.. code-block:: pycon + + >>> import spam + Traceback (most recent call last): + ... + ImportError: dynamic module does not define module export function (PyModExport_spam or PyInit_spam) + + +Module export hook +================== + +The exception you got when you tried to import the module told you that Python +is looking for a "module export function", also known as a +:ref:`module export hook `. +Let's define one. + +First, add a prototype below the ``#include`` lines: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-after: /// Export hook prototype + :end-before: /// + +.. tip:: + The prototype is not strictly necessary, but some modern compilers emit + warnings without it. + It's generally better to add the prototype than to disable the warning. + +The :c:macro:`PyMODEXPORT_FUNC` macro declares the function's +return type, and adds any special linkage declarations needed +to make the function visible and usable when CPython loads it. + +After the prototype, add the function itself. +For now, make it return ``NULL``: + +.. code-block:: c + + PyMODEXPORT_FUNC + PyModExport_spam(void) + { + return NULL; + } + +Compile and load the module again. +You should get a different error this time. + +.. code-block:: pycon + + >>> import spam + Traceback (most recent call last): + ... + SystemError: module export hook for module 'spam' failed without setting an exception + +Simply returning ``NULL`` is *not* correct behavior for an export hook, +and CPython complains about it. +That's good -- it means that CPython found the function! +Let's now make it do something useful. + + +The slot table +============== + +Rather than ``NULL``, the export hook should return the information needed to +create a module. +Let's start with the basics: the name and docstring. + +The information should be defined in an array of +:c:type:`PySlot` entries, which are essentially key-value pairs. +Define this array just before your export hook: + +.. code-block:: c + + PyABIInfo_VAR(abi_info); + + static PySlot spam_slots[] = { + PySlot_STATIC_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_name, "spam"), + PySlot_STATIC_DATA(Py_mod_doc, "A wonderful module with an example function"), + PySlot_END + }; + +The :c:macro:`PySlot_STATIC_DATA` macro is used when the slot value +(here: ``&abi_info``, ``"spam"``, and the docstring) is a pointer to constant, +statically allocated data. + +The ``PyABIInfo_VAR(abi_info);`` macro and the :c:data:`Py_mod_abi` slot +are a bit of boilerplate that helps prevent extensions compiled for +a different version of Python from crashing the interpreter. + +For both :c:data:`Py_mod_name` and :c:data:`Py_mod_doc`, the values are C +strings -- that is, NUL-terminated, UTF-8 encoded byte arrays. + +Note ``PySlot_END`` sentinel entry at the end. +This marks the end of the array. +If you forget it, you'll trigger undefined behavior. + +The array is defined as ``static`` -- that is, not visible outside this ``.c`` file. +This will be a common theme. +CPython only needs to access the export hook; all global variables +and all other functions should generally be ``static``, so that they don't +clash with other extensions. + +Return this array from your export hook instead of ``NULL``: + +.. code-block:: c + :emphasize-lines: 4 + + PyMODEXPORT_FUNC + PyModExport_spam(void) + { + return spam_slots; + } + +Now, recompile and try it out: + +.. code-block:: pycon + + >>> import spam + >>> print(spam) + + +You have an extension module! +Try ``help(spam)`` to see the docstring. + +The next step will be adding a function. + + +.. _backtoexample: + +Exposing a function +=================== + +To expose the :c:func:`system` C function directly to Python, +we'll need to write a layer of glue code to convert arguments from Python +objects to C values, and the C return value back to Python. + +One of the simplest ways to write glue code is a ":c:data:`METH_O`" function, +which takes two Python objects and returns one. +All Python objects -- regardless of the Python type -- are represented in C +as pointers to the :c:type:`PyObject` structure. + +Add such a function above the slots array:: + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + Py_RETURN_NONE; + } + +For now, we ignore the arguments, and use the :c:macro:`Py_RETURN_NONE` +macro, which expands to a ``return`` statement that properly returns +a Python :py:data:`None` object. + +Recompile your extension to make sure you don't have syntax errors. +We haven't yet added ``spam_system`` to the module, so you might get a +warning that ``spam_system`` is unused. + +.. _methodtable: + +Method definitions +------------------ + +To expose the C function to Python, you will need to provide several pieces of +information in a structure called +:c:type:`PyMethodDef` [#why-pymethoddef]_: + +* ``ml_name``: the name of the Python function; +* ``ml_doc``: a docstring; +* ``ml_meth``: the C function to be called; and +* ``ml_flags``: a set of flags describing details like how Python arguments are + passed to the C function. + We'll use :c:data:`METH_O` here -- the flag that matches our + ``spam_system`` function's signature. + +Because modules typically create several functions, these definitions +need to be collected in an array, with a zero-filled sentinel at the end. +Add this array just below the ``spam_system`` function: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-after: /// Module method table + :end-before: /// + +As with module slots, a zero-filled sentinel marks the end of the array. + +Next, we'll add the method to the module. +Add a :c:data:`Py_mod_methods` slot to your :c:type:`PyMethodDef` array: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-after: /// Module slot table + :end-before: /// + :emphasize-lines: 5 + +Recompile your extension again, and test it. +Be sure to restart the Python interpreter, so that ``import spam`` picks +up the new version of the module. + +You should now be able to call the function: + +.. code-block:: pycon + + >>> import spam + >>> print(spam.system) + + >>> print(spam.system('whoami')) + None + +Note that our ``spam.system`` does not yet run the ``whoami`` command; +it only returns ``None``. + +Check that the function accepts exactly one argument, as specified by +the :c:data:`METH_O` flag: + +.. code-block:: pycon + + >>> print(spam.system('too', 'many', 'arguments')) + Traceback (most recent call last): + ... + TypeError: spam.system() takes exactly one argument (3 given) + + +Returning an integer +==================== + +Now, let's take a look at the return value. +Instead of ``None``, we'll want ``spam.system`` to return a number -- that is, +a Python :py:type:`int` object. +Eventually this will be the exit code of a system command, +but let's start with a fixed value, say, ``3``. + +The Python C API provides a function to create a Python :py:type:`int` object +from a C ``int`` value: :c:func:`PyLong_FromLong`. [#why-pylongfromlong]_ + +To call it, replace the ``Py_RETURN_NONE`` with the following 3 lines: + +.. this could be a one-liner, but we want to show the data types here + +.. code-block:: c + :emphasize-lines: 4-6 + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + int status = 3; + PyObject *result = PyLong_FromLong(status); + return result; + } + + +Recompile, restart the Python interpreter again, +and check that the function now returns 3: + +.. code-block:: pycon + + >>> import spam + >>> spam.system('whoami') + 3 + + +Accepting a string +================== + +Finally, let's handle the function argument. + +Our C function, :c:func:`!spam_system`, takes two arguments. +The first one, ``PyObject *self``, will be set to the ``spam`` module +object. +This isn't useful in our case, so we'll ignore it. + +The other one, ``PyObject *arg``, will be set to the object that the user +passed from Python. +We expect that it should be a Python string. +In order to use the information in it, we will need +to convert it to a C value -- in this case, a C string (``const char *``). + +There's a slight type mismatch here: Python's :py:class:`str` objects store +Unicode text, but C strings are arrays of bytes. +So, we'll need to *encode* the data, and we'll use the UTF-8 encoding for it. +(UTF-8 might not always be correct for system commands, but it's what +:py:meth:`str.encode` uses by default, +and the C API has special support for it.) + +The function to encode a Python string into a UTF-8 buffer is named +:c:func:`PyUnicode_AsUTF8AndSize` [#why-pyunicodeasutf8]_. +Call it like this: + +.. code-block:: c + :emphasize-lines: 4 + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + const char *command = PyUnicode_AsUTF8AndSize(arg, NULL); + int status = 3; + PyObject *result = PyLong_FromLong(status); + return result; + } + +If :c:func:`PyUnicode_AsUTF8AndSize` is successful, *command* will point to the +resulting C string -- a zero-terminated array of bytes [#embedded-nul]_. +This buffer is managed by the *arg* object, which means we don't need to free +it, but we must follow some rules: + +* We should only use the buffer inside the ``spam_system`` function. + After ``spam_system`` returns, *arg* and the buffer it manages might be + garbage-collected. +* We must not modify it. This is why we use ``const``. + +If :c:func:`PyUnicode_AsUTF8AndSize` was *not* successful, it returns a ``NULL`` +pointer. +When calling *any* Python C API, we always need to handle such error cases. +The way to do this in general is left for later chapters of this documentation. +For now, be assured that we are already handling errors from +:c:func:`PyLong_FromLong` correctly. + +For the :c:func:`PyUnicode_AsUTF8AndSize` call, the correct way to handle +errors is returning ``NULL`` from ``spam_system``. +Add an ``if`` block for this: + + +.. code-block:: c + :emphasize-lines: 5-7 + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + const char *command = PyUnicode_AsUTF8AndSize(arg); + if (command == NULL) { + return NULL; + } + int status = 3; + PyObject *result = PyLong_FromLong(status); + return result; + } + +To test that error handling works, compile again, restart Python so that +``import spam`` picks up the new version of your module, and try passing +a non-string value to your function: + +.. code-block:: pycon + + >>> import spam + >>> spam.system(3) + Traceback (most recent call last): + ... + TypeError: bad argument type for built-in operation + +Now, all that is left is calling the C library function :c:func:`system` with +the ``char *`` buffer, and using its result instead of the ``3``: + +.. code-block:: c + :emphasize-lines: 8 + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + const char *command = PyUnicode_AsUTF8AndSize(arg); + if (command == NULL) { + return NULL; + } + int status = system(command); + PyObject *result = PyLong_FromLong(status); + return result; + } + +Compile your module, restart Python, and test. +This time, you should see your username -- the output of the ``whoami`` +system command: + +.. code-block:: pycon + + >>> import spam + >>> result = spam.system('whoami') + User Name + >>> result + 0 + +You can also test with other commands, like ``ls``, ``dir``, or one +that doesn't exist: + +.. code-block:: pycon + + >>> import spam + >>> result = spam.system('nonexistent-command') + sh: line 1: nonexistent-command: command not found + >>> result + 32512 + + +The result +========== + + +Congratulations! +You have written a complete Python C API extension module, +and completed this tutorial! + +Here is the entire source file, for your convenience: + +.. _extending-spammodule-source: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-at: /// + + +.. _first-extension-other-tools: + +Appendix: Other build tools +=========================== + +You should be able to follow this tutorial -- except the +*Running your build tool* section itself -- with a build tool other +than ``meson-python``. + +The Python Packaging User Guide has a `list of recommended tools `_; +be sure to choose one for the C language. + + +Workaround for missing PyInit function +-------------------------------------- + +If your build tool output complains about missing ``PyInit_spam``, +add the following function to your module for now: + +.. code-block:: c + + // A workaround + void *PyInit_spam(void) { return NULL; } + +This is a shim for an old-style :ref:`initialization function `, +which was required in extension modules for CPython 3.14 and below. +Current CPython does not need it, but some build tools may still assume that +all extension modules need to define it. + +If you use this workaround, you will get the exception +``SystemError: initialization of spam failed without raising an exception`` +instead of +``ImportError: dynamic module does not define module export function``. + + +Compiling directly +------------------ + +Using a third-party build tool is heavily recommended, +as it will take care of various details of your platform and Python +installation, of naming the resulting extension, and, later, of distributing +your work. + +If you are building an extension for as *specific* system, or for yourself +only, you might instead want to run your compiler directly. +The way to do this is system-specific; be prepared for issues you will need +to solve yourself. + +Linux +^^^^^ + +On Linux, the Python development package may include a ``python3-config`` +command that prints out the required compiler flags. +If you use it, check that it corresponds to the CPython interpreter you'll use +to load the module. +Then, start with the following command: + +.. code-block:: sh + + gcc --shared $(python3-config --cflags --ldflags) spammodule.c -o spam.so + +This should generate a ``spam.so`` file that you need to put in a directory +on :py:attr:`sys.path`. + + +.. rubric:: Footnotes + +.. [#why-spam] ``spam`` is the favorite food of Monty Python fans... +.. [#why-spammodule] The source file name is entirely up to you, + though some tools can be picky about the ``.c`` extension. + This tutorial uses the traditional ``*module.c`` suffix. + Some people would just use :file:`spam.c` to implement a module + named ``spam``, + projects where Python isn't the primary language might use ``py_spam.c``, + and so on. +.. [#stdlib-h] Including :file:`stdlib.h` is technically not necessary, + since :file:`Python.h` includes it and + :ref:`several other standard headers ` for its own use + or for backwards compatibility. + However, it is good practice to explicitly include what you need. +.. [#why-pymethoddef] The :c:type:`!PyMethodDef` structure is also used + to create methods of classes, so there's no separate + ":c:type:`!PyFunctionDef`". +.. [#why-pylongfromlong] The name :c:func:`PyLong_FromLong` + might not seem obvious. + ``PyLong`` refers to a the Python :py:class:`int`, which was originally + called ``long``; the ``FromLong`` refers to the C ``long`` (or ``long int``) + type. +.. [#why-pyunicodeasutf8] Here, ``PyUnicode`` refers to the original name of + the Python :py:class:`str` class: ``unicode``. + + The ``AndSize`` part of the name refers to the fact that this function can + also retrieve the size of the buffer, using an output argument. + We don't need this, so we set the second argument to NULL. +.. [#embedded-nul] We're ignoring the fact that Python strings can also + contain NUL bytes, which terminate a C string. + In other words, our function will treat ``spam.system("foo\0bar")`` as + ``spam.system("foo")``. + This possibility can lead to security issues, so the real ``os.system`` + function size checks for this case and raises an error. diff --git a/Doc/extending/index.rst b/Doc/extending/index.rst index 4cc2c96d8d5b47e..c0c494c3059d99c 100644 --- a/Doc/extending/index.rst +++ b/Doc/extending/index.rst @@ -5,15 +5,17 @@ ################################################## This document describes how to write modules in C or C++ to extend the Python -interpreter with new modules. Those modules can not only define new functions -but also new object types and their methods. The document also describes how +interpreter with new modules. Those modules can do what Python code does -- +define functions, object types and methods -- but also interact with native +libraries or achieve better performance by avoiding the overhead of an +interpreter. The document also describes how to embed the Python interpreter in another application, for use as an extension language. Finally, it shows how to compile and link extension modules so that they can be loaded dynamically (at run time) into the interpreter, if the underlying operating system supports this feature. -This document assumes basic knowledge about Python. For an informal -introduction to the language, see :ref:`tutorial-index`. :ref:`reference-index` +This document assumes basic knowledge about C and Python. For an informal +introduction to Python, see :ref:`tutorial-index`. :ref:`reference-index` gives a more formal definition of the language. :ref:`library-index` documents the existing object types, functions and modules (both built-in and written in Python) that give the language its wide application range. @@ -21,37 +23,75 @@ Python) that give the language its wide application range. For a detailed description of the whole Python/C API, see the separate :ref:`c-api-index`. +To support extensions, Python's C API (Application Programmers Interface) +defines a set of functions, macros and variables that provide access to most +aspects of the Python run-time system. The Python API is incorporated in a C +source file by including the header ``"Python.h"``. + +.. note:: + + The C extension interface is specific to CPython, and extension modules do + not work on other Python implementations. In many cases, it is possible to + avoid writing C extensions and preserve portability to other implementations. + For example, if your use case is calling C library functions or system calls, + you should consider using the :mod:`ctypes` module or the `cffi + `_ library rather than writing + custom C code. + These modules let you write Python code to interface with C code and are more + portable between implementations of Python than writing and compiling a C + extension module. + + +.. toctree:: + :hidden: + + first-extension-module.rst + extending.rst + newtypes_tutorial.rst + newtypes.rst + building.rst + windows.rst + embedding.rst + Recommended third party tools ============================= -This guide only covers the basic tools for creating extensions provided +This document only covers the basic tools for creating extensions provided as part of this version of CPython. Some :ref:`third party tools ` offer both simpler and more sophisticated approaches to creating C and C++ extensions for Python. +While this document is aimed at extension authors, it should also be helpful to +the authors of such tools. +For example, the tutorial module can serve as a simple test case for a build +tool or sample expected output of a code generator. + + +C API Tutorial +============== + +This tutorial describes how to write a simple module in C or C++, +using the Python C API -- that is, using the basic tools provided +as part of this version of CPython. -Creating extensions without third party tools -============================================= + +#. :ref:`first-extension-module` + + +Guides for intermediate topics +============================== This section of the guide covers creating C and C++ extensions without assistance from third party tools. It is intended primarily for creators of those tools, rather than being a recommended way to create your own C extensions. -.. seealso:: - - :pep:`489` -- Multi-phase extension module initialization - -.. toctree:: - :maxdepth: 2 - :numbered: - - extending.rst - newtypes_tutorial.rst - newtypes.rst - building.rst - windows.rst +* :ref:`extending-intro` +* :ref:`defining-new-types` +* :ref:`new-types-topics` +* :ref:`building` +* :ref:`building-on-windows` Embedding the CPython runtime in a larger application ===================================================== @@ -61,8 +101,4 @@ interpreter as the main application, it is desirable to instead embed the CPython runtime inside a larger application. This section covers some of the details involved in doing that successfully. -.. toctree:: - :maxdepth: 2 - :numbered: - - embedding.rst +* :ref:`embedding` diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index e3612f3a1875cac..26085b5cebd3add 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -560,6 +560,8 @@ For an object to be weakly referenceable, the extension type must set the field. The legacy :c:member:`~PyTypeObject.tp_weaklistoffset` field should be left as zero. +If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` should also be set. + Concretely, here is how the statically declared type object would look:: static PyTypeObject TrivialType = { diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index 3bbee33bd50698f..9f3cd1d6f4cf33a 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -6,11 +6,6 @@ Defining Extension Types: Tutorial ********************************** -.. sectionauthor:: Michael Hudson -.. sectionauthor:: Dave Kuhlman -.. sectionauthor:: Jim Fulton - - Python allows the writer of a C extension module to define new types that can be manipulated from Python code, much like the built-in :class:`str` and :class:`list` types. The code for all extension types follows a diff --git a/Doc/extending/windows.rst b/Doc/extending/windows.rst index a97c6182553c302..cd81b443603d174 100644 --- a/Doc/extending/windows.rst +++ b/Doc/extending/windows.rst @@ -47,9 +47,6 @@ things manually, it may be instructive to study the project file for the Differences Between Unix and Windows ==================================== -.. sectionauthor:: Chris Phoenix - - Unix and Windows use completely different paradigms for run-time loading of code. Before you try to build a module that can be dynamically loaded, be aware of how your system works. @@ -109,9 +106,6 @@ separate copy. Using DLLs in Practice ====================== -.. sectionauthor:: Chris Phoenix - - Windows Python is built in Microsoft Visual C++; using other compilers may or may not work. The rest of this section is MSVC++ specific. diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index c758c019ca437b5..ac0aa81e56bb07a 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -591,9 +591,9 @@ exhaustive test suites that exercise every line of code in a module. An appropriate testing discipline can help build large complex applications in Python as well as having interface specifications would. In fact, it can be better because an interface specification cannot test certain properties of a -program. For example, the :meth:`!list.append` method is expected to add new elements +program. For example, the :meth:`list.append` method is expected to add new elements to the end of some internal list; an interface specification cannot test that -your :meth:`!list.append` implementation will actually do this correctly, but it's +your :meth:`list.append` implementation will actually do this correctly, but it's trivial to check this property in a test suite. Writing test suites is very helpful, and you might want to design your code to diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 578777d7f236211..698f2117ff5c649 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -186,7 +186,7 @@ How do I get documentation on Python? ------------------------------------- The standard documentation for the current stable version of Python is available -at https://docs.python.org/3/. PDF, plain text, and downloadable HTML versions are +at https://docs.python.org/3/. EPUB, plain text, and downloadable HTML versions are also available at https://docs.python.org/3/download.html. The documentation is written in reStructuredText and processed by `the Sphinx diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 9f9e4fab685b19f..c2f8f72ee1f2c4b 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -8,11 +8,11 @@ Programming FAQ .. contents:: -General Questions +General questions ================= -Is there a source code level debugger with breakpoints, single-stepping, etc.? ------------------------------------------------------------------------------- +Is there a source code-level debugger with breakpoints and single-stepping? +--------------------------------------------------------------------------- Yes. @@ -25,8 +25,7 @@ Reference Manual `. You can also write your own debugger by using the code for pdb as an example. The IDLE interactive development environment, which is part of the standard -Python distribution (normally available as -`Tools/scripts/idle3 `_), +Python distribution (normally available as :mod:`idlelib`), includes a graphical debugger. PythonWin is a Python IDE that includes a GUI debugger based on pdb. The @@ -48,7 +47,6 @@ There are a number of commercial Python IDEs that include graphical debuggers. They include: * `Wing IDE `_ -* `Komodo IDE `_ * `PyCharm `_ @@ -57,13 +55,15 @@ Are there tools to help find bugs or perform static analysis? Yes. -`Pylint `_ and -`Pyflakes `_ do basic checking that will +`Ruff `__, +`Pylint `__ and +`Pyflakes `__ do basic checking that will help you catch bugs sooner. -Static type checkers such as `Mypy `_, -`Pyre `_, and -`Pytype `_ can check type hints in Python +Static type checkers such as `mypy `__, +`ty `__, +`Pyrefly `__, and +`pytype `__ can check type hints in Python source code. @@ -79,7 +79,7 @@ set of modules required by a program and bind these modules together with a Python binary to produce a single executable. One is to use the freeze tool, which is included in the Python source tree as -`Tools/freeze `_. +:source:`Tools/freeze`. It converts Python byte code to C arrays; with a C compiler you can embed all your modules into a new program, which is then linked with the standard Python modules. @@ -103,6 +103,7 @@ executables: * `py2app `_ (macOS only) * `py2exe `_ (Windows only) + Are there coding standards or a style guide for Python programs? ---------------------------------------------------------------- @@ -110,7 +111,7 @@ Yes. The coding style required for standard library modules is documented as :pep:`8`. -Core Language +Core language ============= .. _faq-unboundlocalerror: @@ -143,7 +144,7 @@ results in an :exc:`!UnboundLocalError`: >>> foo() Traceback (most recent call last): ... - UnboundLocalError: local variable 'x' referenced before assignment + UnboundLocalError: cannot access local variable 'x' where it is not associated with a value This is because when you make an assignment to a variable in a scope, that variable becomes local to that scope and shadows any similarly named variable @@ -208,7 +209,7 @@ Why do lambdas defined in a loop with different values all return the same resul ---------------------------------------------------------------------------------- Assume you use a for loop to define a few different lambdas (or even plain -functions), e.g.:: +functions), for example:: >>> squares = [] >>> for x in range(5): @@ -227,7 +228,7 @@ they all return ``16``:: This happens because ``x`` is not local to the lambdas, but is defined in the outer scope, and it is accessed when the lambda is called --- not when it is defined. At the end of the loop, the value of ``x`` is ``4``, so all the -functions now return ``4**2``, i.e. ``16``. You can also verify this by +functions now return ``4**2``, that is ``16``. You can also verify this by changing the value of ``x`` and see how the results of the lambdas change:: >>> x = 8 @@ -298,9 +299,9 @@ using multiple imports per line uses less screen space. It's good practice if you import modules in the following order: -1. standard library modules -- e.g. :mod:`sys`, :mod:`os`, :mod:`argparse`, :mod:`re` +1. standard library modules -- such as :mod:`sys`, :mod:`os`, :mod:`argparse`, :mod:`re` 2. third-party library modules (anything installed in Python's site-packages - directory) -- e.g. :mod:`!dateutil`, :mod:`!requests`, :mod:`!PIL.Image` + directory) -- such as :pypi:`dateutil`, :pypi:`requests`, :pypi:`tzdata` 3. locally developed modules It is sometimes necessary to move imports to a function or class to avoid @@ -454,7 +455,7 @@ There are two factors that produce this result: (the list), and both ``x`` and ``y`` refer to it. 2) Lists are :term:`mutable`, which means that you can change their content. -After the call to :meth:`!append`, the content of the mutable object has +After the call to :meth:`~sequence.append`, the content of the mutable object has changed from ``[]`` to ``[10]``. Since both the variables refer to the same object, using either name accesses the modified value ``[10]``. @@ -494,11 +495,11 @@ new objects). In other words: -* If we have a mutable object (:class:`list`, :class:`dict`, :class:`set`, - etc.), we can use some specific operations to mutate it and all the variables +* If we have a mutable object (such as :class:`list`, :class:`dict`, :class:`set`), + we can use some specific operations to mutate it and all the variables that refer to it will see the change. -* If we have an immutable object (:class:`str`, :class:`int`, :class:`tuple`, - etc.), all the variables that refer to it will always see the same value, +* If we have an immutable object (such as :class:`str`, :class:`int`, :class:`tuple`), + all the variables that refer to it will always see the same value, but operations that transform that value into a new value always return a new object. @@ -511,7 +512,7 @@ How do I write a function with output parameters (call by reference)? Remember that arguments are passed by assignment in Python. Since assignment just creates references to objects, there's no alias between an argument name in -the caller and callee, and so no call-by-reference per se. You can achieve the +the caller and callee, and consequently no call-by-reference. You can achieve the desired effect in a number of ways. 1) By returning a tuple of the results:: @@ -714,8 +715,8 @@ not:: "a" in ("b", "a") -The same is true of the various assignment operators (``=``, ``+=`` etc). They -are not truly operators but syntactic delimiters in assignment statements. +The same is true of the various assignment operators (``=``, ``+=``, and so on). +They are not truly operators but syntactic delimiters in assignment statements. Is there an equivalent of C's "?:" ternary operator? @@ -868,9 +869,9 @@ with either a space or parentheses. How do I convert a string to a number? -------------------------------------- -For integers, use the built-in :func:`int` type constructor, e.g. ``int('144') +For integers, use the built-in :func:`int` type constructor, for example, ``int('144') == 144``. Similarly, :func:`float` converts to a floating-point number, -e.g. ``float('144') == 144.0``. +for example, ``float('144') == 144.0``. By default, these interpret the number as decimal, so that ``int('0144') == 144`` holds true, and ``int('0x144')`` raises :exc:`ValueError`. ``int(string, @@ -887,18 +888,18 @@ unwanted side effects. For example, someone could pass directory. :func:`eval` also has the effect of interpreting numbers as Python expressions, -so that e.g. ``eval('09')`` gives a syntax error because Python does not allow +so that, for example, ``eval('09')`` gives a syntax error because Python does not allow leading '0' in a decimal number (except '0'). How do I convert a number to a string? -------------------------------------- -To convert, e.g., the number ``144`` to the string ``'144'``, use the built-in type +For example, to convert the number ``144`` to the string ``'144'``, use the built-in type constructor :func:`str`. If you want a hexadecimal or octal representation, use the built-in functions :func:`hex` or :func:`oct`. For fancy formatting, see -the :ref:`f-strings` and :ref:`formatstrings` sections, -e.g. ``"{:04d}".format(144)`` yields +the :ref:`f-strings` and :ref:`formatstrings` sections. +For example, ``"{:04d}".format(144)`` yields ``'0144'`` and ``"{:.3f}".format(1.0/3.0)`` yields ``'0.333'``. @@ -908,7 +909,7 @@ How do I modify a string in place? You can't, because strings are immutable. In most situations, you should simply construct a new string from the various parts you want to assemble it from. However, if you need an object with the ability to modify in-place -unicode data, try using an :class:`io.StringIO` object or the :mod:`array` +Unicode data, try using an :class:`io.StringIO` object or the :mod:`array` module:: >>> import io @@ -1066,13 +1067,14 @@ the raw string:: Also see the specification in the :ref:`language reference `. + Performance =========== My program is too slow. How do I speed it up? --------------------------------------------- -That's a tough one, in general. First, here are a list of things to +That's a tough one, in general. First, here is a list of things to remember before diving further: * Performance characteristics vary across Python implementations. This FAQ @@ -1125,6 +1127,7 @@ yourself. The wiki page devoted to `performance tips `_. + .. _efficient_string_concatenation: What is the most efficient way to concatenate many strings together? @@ -1143,7 +1146,7 @@ them into a list and call :meth:`str.join` at the end:: chunks.append(s) result = ''.join(chunks) -(another reasonably efficient idiom is to use :class:`io.StringIO`) +(Another reasonably efficient idiom is to use :class:`io.StringIO`.) To accumulate many :class:`bytes` objects, the recommended idiom is to extend a :class:`bytearray` object using in-place concatenation (the ``+=`` operator):: @@ -1153,7 +1156,7 @@ a :class:`bytearray` object using in-place concatenation (the ``+=`` operator):: result += b -Sequences (Tuples/Lists) +Sequences (tuples/lists) ======================== How do I convert between tuples and lists? @@ -1217,8 +1220,8 @@ list, deleting duplicates as you go:: else: last = mylist[i] -If all elements of the list may be used as set keys (i.e. they are all -:term:`hashable`) this is often faster :: +If all elements of the list may be used as set keys (that is, they are all +:term:`hashable`) this is often faster:: mylist = list(set(mylist)) @@ -1226,13 +1229,13 @@ This converts the list into a set, thereby removing duplicates, and then back into a list. -How do you remove multiple items from a list --------------------------------------------- +How do you remove multiple items from a list? +--------------------------------------------- As with removing duplicates, explicitly iterating in reverse with a delete condition is one possibility. However, it is easier and faster to use slice replacement with an implicit or explicit forward iteration. -Here are three variations.:: +Here are three variations:: mylist[:] = filter(keep_function, mylist) mylist[:] = (x for x in mylist if keep_condition) @@ -1254,7 +1257,7 @@ difference is that a Python list can contain objects of many different types. The ``array`` module also provides methods for creating arrays of fixed types with compact representations, but they are slower to index than lists. Also note that `NumPy `_ -and other third party packages define array-like structures with +and other third-party packages define array-like structures with various characteristics as well. To get Lisp-style linked lists, you can emulate *cons cells* using tuples:: @@ -1324,7 +1327,7 @@ Or, you can use an extension that provides a matrix datatype; `NumPy How do I apply a method or function to a sequence of objects? ------------------------------------------------------------- -To call a method or function and accumulate the return values is a list, +To call a method or function and accumulate the return values in a list, a :term:`list comprehension` is an elegant solution:: result = [obj.method() for obj in mylist] @@ -1340,6 +1343,7 @@ a plain :keyword:`for` loop will suffice:: for obj in mylist: function(obj) + .. _faq-augmented-assignment-tuple-error: Why does a_tuple[i] += ['item'] raise an exception when the addition works? @@ -1397,9 +1401,9 @@ To see why this happens, you need to know that (a) if an object implements an :meth:`~object.__iadd__` magic method, it gets called when the ``+=`` augmented assignment is executed, and its return value is what gets used in the assignment statement; -and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`!extend` on the list -and returning the list. That's why we say that for lists, ``+=`` is a -"shorthand" for :meth:`!list.extend`:: +and (b) for lists, :meth:`!__iadd__` is equivalent to calling +:meth:`~sequence.extend` on the list and returning the list. +That's why we say that for lists, ``+=`` is a "shorthand" for :meth:`list.extend`:: >>> a_list = [] >>> a_list += [1] @@ -1444,7 +1448,7 @@ How can I sort one list by values from another list? ---------------------------------------------------- Merge them into an iterator of tuples, sort the resulting list, and then pick -out the element you want. :: +out the element you want. >>> list1 = ["what", "I'm", "sorting", "by"] >>> list2 = ["something", "else", "to", "sort"] @@ -1504,14 +1508,15 @@ How do I check if an object is an instance of a given class or of a subclass of Use the built-in function :func:`isinstance(obj, cls) `. You can check if an object is an instance of any of a number of classes by providing a tuple instead of a -single class, e.g. ``isinstance(obj, (class1, class2, ...))``, and can also -check whether an object is one of Python's built-in types, e.g. +single class, for example, ``isinstance(obj, (class1, class2, ...))``, and can also +check whether an object is one of Python's built-in types, for example, ``isinstance(obj, str)`` or ``isinstance(obj, (int, float, complex))``. Note that :func:`isinstance` also checks for virtual inheritance from an :term:`abstract base class`. So, the test will return ``True`` for a registered class even if hasn't directly or indirectly inherited from it. To -test for "true inheritance", scan the :term:`MRO` of the class: +test for "true inheritance", scan the :term:`method resolution order` (MRO) of +the class: .. testcode:: @@ -1574,7 +1579,7 @@ call it:: What is delegation? ------------------- -Delegation is an object oriented technique (also called a design pattern). +Delegation is an object-oriented technique (also called a design pattern). Let's say you have an object ``x`` and want to change the behaviour of just one of its methods. You can create a new class that provides a new implementation of the method you're interested in changing and delegates all other methods to @@ -1645,7 +1650,7 @@ How can I organize my code to make it easier to change the base class? You could assign the base class to an alias and derive from the alias. Then all you have to change is the value assigned to the alias. Incidentally, this trick -is also handy if you want to decide dynamically (e.g. depending on availability +is also handy if you want to decide dynamically (such as depending on availability of resources) which base class to use. Example:: class Base: @@ -1710,9 +1715,9 @@ How can I overload constructors (or methods) in Python? This answer actually applies to all methods, but the question usually comes up first in the context of constructors. -In C++ you'd write +In C++ you'd write: -.. code-block:: c +.. code-block:: c++ class C { C() { cout << "No arguments\n"; } @@ -1731,7 +1736,7 @@ default arguments. For example:: This is not entirely equivalent, but close enough in practice. -You could also try a variable-length argument list, e.g. :: +You could also try a variable-length argument list, for example:: def __init__(self, *args): ... @@ -1774,6 +1779,7 @@ to use private variable names at all. The :ref:`private name mangling specifications ` for details and special cases. + My class defines __del__ but it is not called when I delete the object. ----------------------------------------------------------------------- @@ -1783,7 +1789,7 @@ The :keyword:`del` statement does not necessarily call :meth:`~object.__del__` - decrements the object's reference count, and if this reaches zero :meth:`!__del__` is called. -If your data structures contain circular links (e.g. a tree where each child has +If your data structures contain circular links (for example, a tree where each child has a parent reference and each parent has a list of children) the reference counts will never go back to zero. Once in a while Python runs an algorithm to detect such cycles, but the garbage collector might run some time after the last @@ -1852,6 +1858,8 @@ to the object: 13891296 +.. _faq-identity-with-is: + When can I rely on identity tests with the *is* operator? --------------------------------------------------------- @@ -1883,9 +1891,9 @@ are preferred. In particular, identity tests should not be used to check constants such as :class:`int` and :class:`str` which aren't guaranteed to be singletons:: - >>> a = 1000 - >>> b = 500 - >>> c = b + 500 + >>> a = 10_000_000 + >>> b = 5_000_000 + >>> c = b + 5_000_000 >>> a is c False @@ -1916,7 +1924,7 @@ correctly using identity tests: .. code-block:: python - _sentinel = object() + _sentinel = sentinel('_sentinel') def pop(self, key, default=_sentinel): if key in self: @@ -1954,9 +1962,9 @@ parent class: .. testcode:: - from datetime import date + import datetime as dt - class FirstOfMonthDate(date): + class FirstOfMonthDate(dt.date): "Always choose the first day of the month" def __new__(cls, year, month, day): return super().__new__(cls, year, month, 1) @@ -1995,11 +2003,11 @@ How do I cache method calls? ---------------------------- The two principal tools for caching methods are -:func:`functools.cached_property` and :func:`functools.lru_cache`. The +:deco:`functools.cached_property` and :deco:`functools.lru_cache`. The former stores results at the instance level and the latter at the class level. -The *cached_property* approach only works with methods that do not take +The ``cached_property`` approach only works with methods that do not take any arguments. It does not create a reference to the instance. The cached method result will be kept only as long as the instance is alive. @@ -2008,7 +2016,7 @@ method result will be released right away. The disadvantage is that if instances accumulate, so too will the accumulated method results. They can grow without bound. -The *lru_cache* approach works with methods that have :term:`hashable` +The ``lru_cache`` approach works with methods that have :term:`hashable` arguments. It creates a reference to the instance unless special efforts are made to pass in weak references. @@ -2042,11 +2050,11 @@ This example shows the various techniques:: # Depends on the station_id, date, and units. The above example assumes that the *station_id* never changes. If the -relevant instance attributes are mutable, the *cached_property* approach +relevant instance attributes are mutable, the ``cached_property`` approach can't be made to work because it cannot detect changes to the attributes. -To make the *lru_cache* approach work when the *station_id* is mutable, +To make the ``lru_cache`` approach work when the *station_id* is mutable, the class needs to define the :meth:`~object.__eq__` and :meth:`~object.__hash__` methods so that the cache can detect relevant attribute updates:: @@ -2092,10 +2100,10 @@ one user but run as another, such as if you are testing with a web server. Unless the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable is set, creation of a .pyc file is automatic if you're importing a module and Python -has the ability (permissions, free space, etc...) to create a ``__pycache__`` +has the ability (permissions, free space, and so on) to create a ``__pycache__`` subdirectory and write the compiled module to that subdirectory. -Running Python on a top level script is not considered an import and no +Running Python on a top-level script is not considered an import and no ``.pyc`` will be created. For example, if you have a top-level module ``foo.py`` that imports another module ``xyz.py``, when you run ``foo`` (by typing ``python foo.py`` as a shell command), a ``.pyc`` will be created for @@ -2114,7 +2122,7 @@ the ``compile()`` function in that module interactively:: This will write the ``.pyc`` to a ``__pycache__`` subdirectory in the same location as ``foo.py`` (or you can override that with the optional parameter -``cfile``). +*cfile*). You can also automatically compile all files in a directory or directories using the :mod:`compileall` module. You can do it from the shell prompt by running @@ -2219,7 +2227,7 @@ changed module, do this:: importlib.reload(modname) Warning: this technique is not 100% fool-proof. In particular, modules -containing statements like :: +containing statements like:: from modname import some_objects diff --git a/Doc/faq/python-video-icon.png b/Doc/faq/python-video-icon.png deleted file mode 100644 index 265da50c7b38fca..000000000000000 Binary files a/Doc/faq/python-video-icon.png and /dev/null differ diff --git a/Doc/glossary.rst b/Doc/glossary.rst index b7bd547d38fd1ec..b25532d2d634126 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -21,7 +21,9 @@ Glossary right delimiters (parentheses, square brackets, curly braces or triple quotes), or after specifying a decorator. - * The :const:`Ellipsis` built-in constant. + .. index:: single: ...; ellipsis literal + + * The three dots form of the :ref:`Ellipsis ` object. abstract base class Abstract base classes complement :term:`duck-typing` by @@ -37,10 +39,11 @@ Glossary ABCs with the :mod:`abc` module. annotate function - A function that can be called to retrieve the :term:`annotations ` - of an object. This function is accessible as the :attr:`~object.__annotate__` - attribute of functions, classes, and modules. Annotate functions are a - subset of :term:`evaluate functions `. + A callable that can be called to retrieve the :term:`annotations ` of + an object. Annotate functions are usually :term:`functions `, + automatically generated as the :attr:`~object.__annotate__` attribute of functions, + classes, and modules. Annotate functions are a subset of + :term:`evaluate functions `. annotation A label associated with a variable, a class @@ -93,21 +96,24 @@ Glossary :meth:`~object.__aexit__` methods. Introduced by :pep:`492`. asynchronous generator - A function which returns an :term:`asynchronous generator iterator`. It - looks like a coroutine function defined with :keyword:`async def` except - that it contains :keyword:`yield` expressions for producing a series of - values usable in an :keyword:`async for` loop. - - Usually refers to an asynchronous generator function, but may refer to an - *asynchronous generator iterator* in some contexts. In cases where the - intended meaning isn't clear, using the full terms avoids ambiguity. + Informally used to mean either an :term:`asynchronous generator + function` or an :term:`asynchronous generator iterator`, depending on + context. The formal terms :term:`asynchronous generator function` and + :term:`asynchronous generator iterator` are uncommon in practice; + "asynchronous generator" alone is almost always sufficient. + + asynchronous generator function + A function which returns an :term:`asynchronous generator iterator`. + It looks like a coroutine function defined with :keyword:`async def` + except that it contains :keyword:`yield` expressions for producing a + series of values usable in an :keyword:`async for` loop. See :pep:`525`. An asynchronous generator function may contain :keyword:`await` expressions as well as :keyword:`async for`, and :keyword:`async with` statements. asynchronous generator iterator - An object created by a :term:`asynchronous generator` function. + An object created by an :term:`asynchronous generator function`. This is an :term:`asynchronous iterator` which when called using the :meth:`~object.__anext__` method returns an awaitable object which will execute @@ -132,6 +138,14 @@ Glossary iterator's :meth:`~object.__anext__` method until it raises a :exc:`StopAsyncIteration` exception. Introduced by :pep:`492`. + atomic operation + An operation that appears to execute as a single, indivisible step: no + other thread can observe it half-done, and its effects become visible all + at once. Python does not guarantee that high-level statements are atomic + (for example, ``x += 1`` performs multiple bytecode operations and is not + atomic). Atomicity is only guaranteed where explicitly documented. See + also :term:`race condition` and :term:`data race`. + attached thread state A :term:`thread state` that is active for the current OS thread. @@ -150,9 +164,9 @@ Glossary On most builds of Python, having an attached thread state implies that the caller holds the :term:`GIL` for the current interpreter, so only one OS thread can have an attached thread state at a given moment. In - :term:`free-threaded ` builds of Python, threads can concurrently - hold an attached thread state, allowing for true parallelism of the bytecode - interpreter. + :term:`free-threaded builds ` of Python, threads can + concurrently hold an attached thread state, allowing for true parallelism of + the bytecode interpreter. attribute A value associated with an object which is usually referenced by name @@ -287,6 +301,22 @@ Glossary advanced mathematical feature. If you're not aware of a need for them, it's almost certain you can safely ignore them. + concurrency + The ability of a computer program to perform multiple tasks at the same + time. Python provides libraries for writing programs that make use of + different forms of concurrency. :mod:`asyncio` is a library for dealing + with asynchronous tasks and coroutines. :mod:`threading` provides + access to operating system threads and :mod:`multiprocessing` to + operating system processes. Multi-core processors can execute threads and + processes on different CPU cores at the same time (see + :term:`parallelism`). + + concurrent modification + When multiple threads modify shared data at the same time. Concurrent + modification without proper synchronization can cause + :term:`race conditions `, and might also trigger a + :term:`data race `, data corruption, or both. + context This term has different meanings depending on where and how it is used. Some common meanings: @@ -361,10 +391,32 @@ Glossary the :term:`cyclic garbage collector ` is to identify these groups and break the reference cycles so that the memory can be reclaimed. + data race + A situation where multiple threads access the same memory location + concurrently, at least one of the accesses is a write, and the threads + do not use any synchronization to control their access. Data races + lead to :term:`non-deterministic` behavior and can cause data corruption. + Proper use of :term:`locks ` and other :term:`synchronization primitives + ` prevents data races. Note that data races + can only happen in native code, but that :term:`native code` might be + exposed in a Python API. See also :term:`race condition` and + :term:`thread-safe`. + + deadlock + A situation in which two or more tasks (threads, processes, or coroutines) + wait indefinitely for each other to release resources or complete actions, + preventing any from making progress. For example, if thread A holds lock + 1 and waits for lock 2, while thread B holds lock 2 and waits for lock 1, + both threads will wait indefinitely. In Python this often arises from + acquiring multiple locks in conflicting orders or from circular + join/await dependencies. Deadlocks can be avoided by always acquiring + multiple :term:`locks ` in a consistent order. See also + :term:`lock` and :term:`reentrant`. + decorator A function returning another function, usually applied as a function transformation using the ``@wrapper`` syntax. Common examples for - decorators are :func:`classmethod` and :func:`staticmethod`. + decorators are :deco:`classmethod` and :deco:`staticmethod`. The decorator syntax is merely syntactic sugar, the following two function definitions are semantically equivalent:: @@ -435,6 +487,11 @@ Glossary with :term:`abstract base classes `.) Instead, it typically employs :func:`hasattr` tests or :term:`EAFP` programming. + dunder + An informal short-hand for "double underscore", used when talking about a + :term:`special method`. For example, ``__init__`` is often pronounced + "dunder init". + EAFP Easier to ask for forgiveness than permission. This common Python coding style assumes the existence of valid keys or attributes and catches @@ -462,6 +519,7 @@ Glossary core and with user code. f-string + f-strings String literals prefixed with ``f`` or ``F`` are commonly called "f-strings" which is short for :ref:`formatted string literals `. See also :pep:`498`. @@ -526,6 +584,13 @@ Glossary the :term:`global interpreter lock` which allows only one thread to execute Python bytecode at a time. See :pep:`703`. + free-threaded build + + A build of :term:`CPython` that supports :term:`free threading`, + configured using the :option:`--disable-gil` option before compilation. + + See :ref:`freethreading-python-howto`. + free variable Formally, as defined in the :ref:`language execution model `, a free variable is any variable used in a namespace which is not a local variable in that @@ -579,23 +644,33 @@ Glossary .. index:: single: generator generator - A function which returns a :term:`generator iterator`. It looks like a - normal function except that it contains :keyword:`yield` expressions - for producing a series of values usable in a for-loop or that can be - retrieved one at a time with the :func:`next` function. + Informally used to mean either a :term:`generator function` or a + :term:`generator iterator`, depending on context. The formal terms + :term:`generator function` and :term:`generator iterator` are uncommon + in practice; "generator" alone is almost always sufficient. - Usually refers to a generator function, but may refer to a - *generator iterator* in some contexts. In cases where the intended - meaning isn't clear, using the full terms avoids ambiguity. + .. index:: single: generator function + + generator function + A function which returns a :term:`generator` object. It looks like a + normal function except that it contains :keyword:`yield` expressions + for producing a series of values usable in a :keyword:`for`\-loop or + that can be retrieved one at a time with the :func:`next` function. + See :ref:`yieldexpr`. generator iterator - An object created by a :term:`generator` function. + An object created by a :term:`generator function` or a + :term:`generator expression`. Each :keyword:`yield` temporarily suspends processing, remembering the - execution state (including local variables and pending - try-statements). When the *generator iterator* resumes, it picks up where - it left off (in contrast to functions which start fresh on every - invocation). + execution state (including local variables and pending try-statements). + When the *generator iterator* resumes, it picks up where it left off + (in contrast to functions which start fresh on every invocation). + + Generator iterators also implement the :meth:`~generator.send` method + to send a value into the suspended generator, and the + :meth:`~generator.throw` method to raise an exception at the point + where the generator was paused. See :ref:`generator-methods`. .. index:: single: generator expression @@ -614,7 +689,7 @@ Glossary determined by the dispatch algorithm. See also the :term:`single dispatch` glossary entry, the - :func:`functools.singledispatch` decorator, and :pep:`443`. + :deco:`functools.singledispatch` decorator, and :pep:`443`. generic type A :term:`type` that can be parameterized; typically a @@ -654,6 +729,14 @@ Glossary requires the GIL to be held in order to use it. This refers to having an :term:`attached thread state`. + global state + Data that is accessible throughout a program, such as module-level + variables, class variables, or C static variables in :term:`extension modules + `. In multi-threaded programs, global state shared + between threads typically requires synchronization to avoid + :term:`race conditions ` and + :term:`data races `. + hash-based pyc A bytecode cache file that uses the hash rather than the last-modified time of the corresponding source file to determine its validity. See @@ -698,7 +781,9 @@ Glossary tuples. Such an object cannot be altered. A new object has to be created if a different value has to be stored. They play an important role in places where a constant hash value is needed, for example as a key - in a dictionary. + in a dictionary. Immutable objects are inherently :term:`thread-safe` + because their state cannot be modified after creation, eliminating concerns + about improperly synchronized :term:`concurrent modification`. import path A list of locations (or :term:`path entries `) that are @@ -715,6 +800,19 @@ Glossary An object that both finds and loads a module; both a :term:`finder` and :term:`loader` object. + index + A numeric value that represents the position of an element in + a :term:`sequence`. + + In Python, indexing starts at zero. + For example, ``things[0]`` names the *first* element of ``things``; + ``things[1]`` names the second one. + + In some contexts, Python allows negative indexes for counting from the + end of a sequence, and indexing using :term:`slices `. + + See also :term:`subscript`. + interactive Python has an interactive interpreter which means you can enter statements and expressions at the interpreter prompt, immediately @@ -788,9 +886,13 @@ Glossary CPython does not consistently apply the requirement that an iterator define :meth:`~iterator.__iter__`. - And also please note that the free-threading CPython does not guarantee - the thread-safety of iterator operations. + And also please note that :term:`free-threaded ` + CPython does not guarantee :term:`thread-safe` behavior of iterator + operations. + key + A value that identifies an entry in a :term:`mapping`. + See also :term:`subscript`. key function A key function or collation function is a callable that returns a value @@ -805,7 +907,7 @@ Glossary :func:`itertools.groupby`. There are several ways to create a key function. For example. the - :meth:`str.lower` method can serve as a key function for case insensitive + :meth:`str.casefold` method can serve as a key function for case insensitive sorts. Alternatively, a key function can be built from a :keyword:`lambda` expression such as ``lambda r: (r[0], r[2])``. Also, :func:`operator.attrgetter`, :func:`operator.itemgetter`, and @@ -827,10 +929,11 @@ Glossary :keyword:`if` statements. In a multi-threaded environment, the LBYL approach can risk introducing a - race condition between "the looking" and "the leaping". For example, the - code, ``if key in mapping: return mapping[key]`` can fail if another + :term:`race condition` between "the looking" and "the leaping". For example, + the code, ``if key in mapping: return mapping[key]`` can fail if another thread removes *key* from *mapping* after the test, but before the lookup. - This issue can be solved with locks or by using the EAFP approach. + This issue can be solved with :term:`locks ` or by using the + :term:`EAFP` approach. See also :term:`thread-safe`. lexical analyzer @@ -849,6 +952,29 @@ Glossary clause is optional. If omitted, all elements in ``range(256)`` are processed. + lock + A :term:`synchronization primitive` that allows only one thread at a + time to access a shared resource. A thread must acquire a lock before + accessing the protected resource and release it afterward. If a thread + attempts to acquire a lock that is already held by another thread, it + will block until the lock becomes available. Python's :mod:`threading` + module provides :class:`~threading.Lock` (a basic lock) and + :class:`~threading.RLock` (a :term:`reentrant` lock). Locks are used + to prevent :term:`race conditions ` and ensure + :term:`thread-safe` access to shared data. Alternative design patterns + to locks exist such as queues, producer/consumer patterns, and + thread-local state. See also :term:`deadlock`, and :term:`reentrant`. + + lock-free + An operation that does not acquire any :term:`lock` and uses atomic CPU + instructions to ensure correctness. Lock-free operations can execute + concurrently without blocking each other and cannot be blocked by + operations that hold locks. In :term:`free-threaded ` + Python, built-in types like :class:`dict` and :class:`list` provide + lock-free read operations, which means other threads may observe + intermediate states during multi-step modifications even when those + modifications hold the :term:`per-object lock`. + loader An object that loads a module. It must define the :meth:`!exec_module` and :meth:`!create_module` methods @@ -934,8 +1060,11 @@ Glossary See :term:`method resolution order`. mutable - Mutable objects can change their value but keep their :func:`id`. See - also :term:`immutable`. + An :term:`object` with state that is allowed to change during the course + of the program. In multi-threaded programs, mutable objects that are + shared between threads require careful synchronization to avoid + :term:`race conditions `. See also :term:`immutable`, + :term:`thread-safe`, and :term:`concurrent modification`. named tuple The term "named tuple" applies to any type or class that inherits from @@ -987,6 +1116,13 @@ Glossary See also :term:`module`. + native code + Code that is compiled to machine instructions and runs directly on the + processor, as opposed to code that is interpreted or runs in a virtual + machine. In the context of Python, native code typically refers to + C, C++, Rust or Fortran code in :term:`extension modules ` + that can be called from Python. See also :term:`extension module`. + nested scope The ability to refer to a variable in an enclosing definition. For instance, a function defined inside another function can refer to @@ -1003,6 +1139,15 @@ Glossary properties, :meth:`~object.__getattribute__`, class methods, and static methods. + non-deterministic + Behavior where the outcome of a program can vary between executions with + the same inputs. In multi-threaded programs, non-deterministic behavior + often results from :term:`race conditions ` where the + relative timing or interleaving of threads affects the result. + Proper synchronization using :term:`locks ` and other + :term:`synchronization primitives ` helps + ensure deterministic behavior. + object Any data with state (attributes or value) and defined behavior (methods). Also the ultimate base class of any :term:`new-style @@ -1017,6 +1162,15 @@ Glossary applied to all scopes, only those relying on a known set of local and nonlocal variable names are restricted to optimized scopes. + optional module + An :term:`extension module` that is part of the :term:`standard library`, + but may be absent in some builds of :term:`CPython`, + usually due to missing third-party libraries or because the module + is not available for a given platform. + + See :ref:`optional-module-requirements` for a list of optional modules + that require third-party libraries. + package A Python :term:`module` which can contain submodules or recursively, subpackages. Technically, a package is a Python module with a @@ -1024,6 +1178,16 @@ Glossary See also :term:`regular package` and :term:`namespace package`. + parallelism + Executing multiple operations at the same time (e.g. on multiple CPU + cores). In Python builds with the + :term:`global interpreter lock (GIL) `, only one + thread runs Python bytecode at a time, so taking advantage of multiple + CPU cores typically involves multiple processes + (e.g. :mod:`multiprocessing`) or native extensions that release the GIL. + In :term:`free-threaded ` Python, multiple Python threads + can run Python code simultaneously on different cores. + parameter A named entity in a :term:`function` (or method) definition that specifies an :term:`argument` (or in some cases, arguments) that the @@ -1077,6 +1241,16 @@ Glossary `, the :class:`inspect.Parameter` class, the :ref:`function` section, and :pep:`362`. + per-object lock + A :term:`lock` associated with an individual object instance rather than + a global lock shared across all objects. In :term:`free-threaded + ` Python, built-in types like :class:`dict` and + :class:`list` use per-object locks to allow concurrent operations on + different objects while serializing operations on the same object. + Operations that hold the per-object lock prevent other locking operations + on the same object from proceeding, but do not block :term:`lock-free` + operations. + path entry A single location on the :term:`import path` which the :term:`path based finder` consults to find modules for importing. @@ -1198,6 +1372,18 @@ Glossary >>> email.mime.text.__name__ 'email.mime.text' + race condition + A condition of a program where the behavior + depends on the relative timing or ordering of events, particularly in + multi-threaded programs. Race conditions can lead to + :term:`non-deterministic` behavior and bugs that are difficult to + reproduce. A :term:`data race` is a specific type of race condition + involving unsynchronized access to shared memory. The :term:`LBYL` + coding style is particularly susceptible to race conditions in + multi-threaded code. Using :term:`locks ` and other + :term:`synchronization primitives ` + helps prevent race conditions. + reference count The number of references to an object. When the reference count of an object drops to zero, it is deallocated. Some objects are @@ -1219,6 +1405,25 @@ Glossary See also :term:`namespace package`. + reentrant + A property of a function or :term:`lock` that allows it to be called or + acquired multiple times by the same thread without causing errors or a + :term:`deadlock`. + + For functions, reentrancy means the function can be safely called again + before a previous invocation has completed, which is important when + functions may be called recursively or from signal handlers. Thread-unsafe + functions may be :term:`non-deterministic` if they're called reentrantly in a + multithreaded program. + + For locks, Python's :class:`threading.RLock` (reentrant lock) is + reentrant, meaning a thread that already holds the lock can acquire it + again without blocking. In contrast, :class:`threading.Lock` is not + reentrant - attempting to acquire it twice from the same thread will cause + a deadlock. + + See also :term:`lock` and :term:`deadlock`. + REPL An acronym for the "read–eval–print loop", another name for the :term:`interactive` interpreter shell. @@ -1243,8 +1448,9 @@ Glossary The :class:`collections.abc.Sequence` abstract base class defines a much richer interface that goes beyond just :meth:`~object.__getitem__` and :meth:`~object.__len__`, adding - :meth:`!count`, :meth:`!index`, :meth:`~object.__contains__`, and - :meth:`~object.__reversed__`. Types that implement this expanded + :meth:`~sequence.count`, :meth:`~sequence.index`, + :meth:`~object.__contains__`, and :meth:`~object.__reversed__`. + Types that implement this expanded interface can be registered explicitly using :func:`~abc.ABCMeta.register`. For more documentation on sequence methods generally, see @@ -1261,10 +1467,11 @@ Glossary chosen based on the type of a single argument. slice - An object usually containing a portion of a :term:`sequence`. A slice is - created using the subscript notation, ``[]`` with colons between numbers - when several are given, such as in ``variable_name[1:3:5]``. The bracket - (subscript) notation uses :class:`slice` objects internally. + An object of type :class:`slice`, used to describe a portion of + a :term:`sequence`. + A slice object is created when using the :ref:`slicing ` form + of :ref:`subscript notation `, with colons inside square + brackets, such as in ``variable_name[1:3:5]``. soft deprecated A soft deprecated API should not be used in new code, @@ -1322,7 +1529,28 @@ Glossary See also :term:`borrowed reference`. + subscript + The expression in square brackets of a + :ref:`subscription expression `, for example, + the ``3`` in ``items[3]``. + Usually used to select an element of a container. + Also called a :term:`key` when subscripting a :term:`mapping`, + or an :term:`index` when subscripting a :term:`sequence`. + + synchronization primitive + A basic building block for coordinating (synchronizing) the execution of + multiple threads to ensure :term:`thread-safe` access to shared resources. + Python's :mod:`threading` module provides several synchronization primitives + including :class:`~threading.Lock`, :class:`~threading.RLock`, + :class:`~threading.Semaphore`, :class:`~threading.Condition`, + :class:`~threading.Event`, and :class:`~threading.Barrier`. Additionally, + the :mod:`queue` module provides multi-producer, multi-consumer queues + that are especially useful in multithreaded programs. These + primitives help prevent :term:`race conditions ` and + coordinate thread execution. See also :term:`lock`. + t-string + t-strings String literals prefixed with ``t`` or ``T`` are commonly called "t-strings" which is short for :ref:`template string literals `. @@ -1373,6 +1601,19 @@ Glossary See :ref:`Thread State and the Global Interpreter Lock ` for more information. + thread-safe + A module, function, or class that behaves correctly when used by multiple + threads concurrently. Thread-safe code uses appropriate + :term:`synchronization primitives ` like + :term:`locks ` to protect shared mutable state, or is designed + to avoid shared mutable state entirely. In the + :term:`free-threaded ` build, built-in types like + :class:`dict`, :class:`list`, and :class:`set` use internal locking + to make many operations thread-safe, although thread safety is not + necessarily guaranteed. Code that is not thread-safe may experience + :term:`race conditions ` and :term:`data races ` + when used in multi-threaded programs. + token A small unit of source code, generated by the @@ -1472,6 +1713,11 @@ Glossary A computer defined entirely in software. Python's virtual machine executes the :term:`bytecode` emitted by the bytecode compiler. + walrus operator + A light-hearted way to refer to the :ref:`assignment expression + ` operator ``:=`` because it looks a bit like a + walrus if you turn your head. + Zen of Python Listing of Python design principles and philosophies that are helpful in understanding and using the language. The listing can be found by typing diff --git a/Doc/howto/a-conceptual-overview-of-asyncio.rst b/Doc/howto/a-conceptual-overview-of-asyncio.rst new file mode 100644 index 000000000000000..7a7a87cb9584001 --- /dev/null +++ b/Doc/howto/a-conceptual-overview-of-asyncio.rst @@ -0,0 +1,619 @@ +.. _a-conceptual-overview-of-asyncio: + +**************************************** +A conceptual overview of :mod:`!asyncio` +**************************************** + +This :ref:`HOWTO ` article seeks to help you build a sturdy mental +model of how :mod:`asyncio` fundamentally works, helping you understand the +how and why behind the recommended patterns. + +You might be curious about some key :mod:`!asyncio` concepts. +By the end of this article, you'll be able to comfortably answer these questions: + +- What's happening behind the scenes when an object is awaited? +- How does :mod:`!asyncio` differentiate between a task which doesn't need + CPU time (such as a network request or file read) as opposed to a task that + does (such as computing n-factorial)? +- How to write an asynchronous variant of an operation, such as + an async sleep or database request. + +.. seealso:: + + * The `guide `_ that inspired this HOWTO article, by Alexander Nordin. + * This in-depth `YouTube tutorial series `_ on + ``asyncio`` created by Python core team member, Łukasz Langa. + * `500 Lines or Less: A Web Crawler With asyncio Coroutines `_ by A. + Jesse Jiryu Davis and Guido van Rossum. + +-------------------------------------------- +A conceptual overview part 1: the high-level +-------------------------------------------- + +In part 1, we'll cover the main, high-level building blocks of :mod:`!asyncio`: +the event loop, coroutine functions, coroutine objects, tasks, and ``await``. + +========== +Event loop +========== + +Everything in :mod:`!asyncio` happens relative to the event loop. +It's the star of the show, but prefers to work behind the scenes, managing +and coordinating resources. +It's like an orchestra conductor. +Some power is explicitly granted to it, but a lot of its ability to get things +done comes from the respect and cooperation of its band members. + +In more technical terms, the event loop contains a collection of jobs to be run. +Some jobs are added directly by you, and some indirectly by :mod:`!asyncio`. +The event loop takes a job from its backlog of work and invokes it (or "gives +it control"), similar to calling a function, and then that job runs. +Once it pauses or completes, it returns control to the event loop. +The event loop will then select another job from its pool and invoke it. +You can *roughly* think of the collection of jobs as a queue: jobs are added and +then processed one at a time, generally (but not always) in order. +This process repeats indefinitely, with the event loop cycling endlessly +onwards. +If there are no more jobs pending execution, the event loop is smart enough to +rest and avoid needlessly wasting CPU cycles, and will come back when there's +more work to be done, such as when I/O operations complete or timers expire. + +Effective execution relies on jobs sharing well and cooperating; a greedy job +could hog control and leave the other jobs to starve, rendering the overall +event loop approach rather useless. + +:: + + import asyncio + + # This creates an event loop and indefinitely cycles through + # its collection of jobs. + event_loop = asyncio.new_event_loop() + event_loop.run_forever() + +===================================== +Asynchronous functions and coroutines +===================================== + +This is a basic, boring Python function:: + + def hello_printer(): + print( + "Hi, I am a lowly, simple printer, though I have all I " + "need in life -- \nfresh paper and my dearly beloved octopus " + "partner in crime." + ) + +Calling a regular function invokes its logic or body:: + + >>> hello_printer() + Hi, I am a lowly, simple printer, though I have all I need in life -- + fresh paper and my dearly beloved octopus partner in crime. + +The :ref:`async def `, as opposed to just a plain ``def``, makes +this an asynchronous function (or "coroutine function"). +Calling it creates and returns a :ref:`coroutine ` object. + +:: + + async def loudmouth_penguin(magic_number: int): + print( + "I am a super special talking penguin. Far cooler than that printer. " + f"By the way, my lucky number is: {magic_number}." + ) + +Calling the async function, ``loudmouth_penguin``, does not execute the print statement; +instead, it creates a coroutine object:: + + >>> loudmouth_penguin(magic_number=3) + + +The terms "coroutine function" and "coroutine object" are often conflated +as coroutine. +That can be confusing! +In this article, coroutine specifically refers to a coroutine object, or more +precisely, an instance of :class:`types.CoroutineType` (native coroutine). +Note that coroutines can also exist as instances of +:class:`collections.abc.Coroutine` -- a distinction that matters for type +checking. + +A coroutine represents the function's body or logic. +A coroutine has to be explicitly started; again, merely creating the coroutine +does not start it. +Notably, the coroutine can be paused and resumed at various points within the +function's body. +That pausing and resuming ability is what allows for asynchronous behavior! + +Coroutines and coroutine functions were built by leveraging the functionality +of :term:`generators ` and +:term:`generator functions `. +Recall, a generator function is a function that :keyword:`yield`\s, like this +one:: + + def get_random_number(): + # This would be a bad random number generator! + print("Hi") + yield 1 + print("Hello") + yield 7 + print("Howdy") + yield 4 + ... + +Similar to a coroutine function, calling a generator function does not run it. +Instead, it creates a generator object:: + + >>> get_random_number() + + +You can proceed to the next ``yield`` of a generator by using the +built-in function :func:`next`. +In other words, the generator runs, then pauses. +For example:: + + >>> generator = get_random_number() + >>> next(generator) + Hi + 1 + >>> next(generator) + Hello + 7 + +===== +Tasks +===== + +Roughly speaking, :ref:`tasks ` are coroutines (not coroutine +functions) tied to an event loop. +A task also maintains a list of callback functions whose importance will become +clear in a moment when we discuss :keyword:`await`. + +Creating a task automatically schedules it for execution (by adding a +callback to run it in the event loop's to-do list, that is, collection of jobs). +The recommended way to create tasks is via :func:`asyncio.create_task`. + +:mod:`!asyncio` automatically associates tasks with the event loop for you. +This automatic association was purposely designed into :mod:`!asyncio` for +the sake of simplicity. +Without it, you'd have to keep track of the event loop object and pass it to +any coroutine function that wants to create tasks, adding redundant clutter +to your code. + +:: + + coroutine = loudmouth_penguin(magic_number=5) + # This creates a Task object and schedules its execution via the event loop. + task = asyncio.create_task(coroutine) + +Earlier, we manually created the event loop and set it to run forever. +In practice, it's recommended to use (and common to see) :func:`asyncio.run`, +which takes care of managing the event loop and ensuring the provided +coroutine finishes before advancing. +For example, many async programs follow this setup:: + + import asyncio + + async def main(): + # Perform all sorts of wacky, wild asynchronous things... + ... + + if __name__ == "__main__": + asyncio.run(main()) + # The program will not reach the following print statement until the + # coroutine main() finishes. + print("coroutine main() is done!") + +It's important to be aware that the task itself is not added to the event loop, +only a callback to the task is. +This matters if the task object you created is garbage collected before it's +called by the event loop. +For example, consider this program: + +.. code-block:: + :linenos: + + async def hello(): + print("hello!") + + async def main(): + asyncio.create_task(hello()) + # Other asynchronous instructions which run for a while + # and cede control to the event loop... + ... + + asyncio.run(main()) + +Because there's no reference to the task object created on line 5, it *might* +be garbage collected before the event loop invokes it. +Later instructions in the coroutine ``main()`` hand control back to the event +loop so it can invoke other jobs. +When the event loop eventually tries to run the task, it might fail and +discover the task object does not exist! +This can also happen even if a coroutine keeps a reference to a task but +completes before that task finishes. +When the coroutine exits, local variables go out of scope and may be subject +to garbage collection. +In practice, ``asyncio`` and Python's garbage collector work pretty hard to +ensure this sort of thing doesn't happen. +But that's no reason to be reckless! + +===== +await +===== + +:keyword:`await` is a Python keyword that's commonly used in one of two +different ways:: + + await task + await coroutine + +In a crucial way, the behavior of ``await`` depends on the type of object +being awaited. + +^^^^^^^^^^^^^^ +Awaiting tasks +^^^^^^^^^^^^^^ + +Awaiting a task will cede control from the current task or coroutine to +the event loop. +In the process of relinquishing control, a few important things happen. +We'll use the following code example to illustrate:: + + async def plant_a_tree(): + dig_the_hole_task = asyncio.create_task(dig_the_hole()) + await dig_the_hole_task + + # Other instructions associated with planting a tree. + ... + +In this example, imagine the event loop has passed control to the start of the +coroutine ``plant_a_tree()``. +As seen above, the coroutine creates a task and then awaits it. +The ``await dig_the_hole_task`` instruction adds a callback (which will resume +``plant_a_tree()``) to the ``dig_the_hole_task`` object's list of callbacks. +And then, the instruction cedes control to the event loop. +Some time later, the event loop will pass control to ``dig_the_hole_task`` +and the task will finish whatever it needs to do. +Once the task finishes, it will add its various callbacks to the event loop, +in this case, a call to resume ``plant_a_tree()``. + +Generally speaking, when the awaited task finishes (``dig_the_hole_task``), +the original task or coroutine (``plant_a_tree()``) is added back to the event +loop's to-do list to be resumed. + +This is a basic, yet reliable mental model. +In practice, the control handoffs are slightly more complex, but not by much. +In part 2, we'll walk through the details that make this possible. + +^^^^^^^^^^^^^^^^^^^ +Awaiting coroutines +^^^^^^^^^^^^^^^^^^^ + +**Unlike tasks, awaiting a coroutine does not hand control back to the event +loop!** +Wrapping a coroutine in a task first, then awaiting that would cede +control. +The behavior of ``await coroutine`` is effectively the same as invoking a +regular, synchronous Python function. +Consider this program:: + + import asyncio + + async def coro_a(): + print("I am coro_a(). Hi!") + + async def coro_b(): + print("I am coro_b(). I sure hope no one hogs the event loop...") + + async def main(): + task_b = asyncio.create_task(coro_b()) + num_repeats = 3 + for _ in range(num_repeats): + await coro_a() + await task_b + + asyncio.run(main()) + +The first statement in the coroutine ``main()`` creates ``task_b`` and schedules +it for execution via the event loop. +Then, ``coro_a()`` is repeatedly awaited. Control never cedes to the +event loop, which is why we see the output of all three ``coro_a()`` +invocations before ``coro_b()``'s output: + +.. code-block:: none + + I am coro_a(). Hi! + I am coro_a(). Hi! + I am coro_a(). Hi! + I am coro_b(). I sure hope no one hogs the event loop... + +If we change ``await coro_a()`` to ``await asyncio.create_task(coro_a())``, the +behavior changes. +The coroutine ``main()`` cedes control to the event loop with that statement. +The event loop then proceeds through its backlog of work, calling ``task_b`` +and then the task which wraps ``coro_a()`` before resuming the coroutine +``main()``. + +.. code-block:: none + + I am coro_b(). I sure hope no one hogs the event loop... + I am coro_a(). Hi! + I am coro_a(). Hi! + I am coro_a(). Hi! + +This behavior of ``await coroutine`` can trip a lot of people up! +That example highlights how using only ``await coroutine`` could +unintentionally hog control from other tasks and effectively stall the event +loop. +:func:`asyncio.run` can help you detect such occurrences via the +``debug=True`` flag, which enables +:ref:`debug mode `. +Among other things, it will log any coroutines that monopolize execution for +100ms or longer. + +The design intentionally trades off some conceptual clarity around usage of +``await`` for improved performance. +Each time a task is awaited, control needs to be passed all the way up the +call stack to the event loop. +Then, the event loop needs to manage its internal state and work through +its processing logic to resume the next job. +That might sound minor, but in a large program with many ``await``\ s, that +overhead can add up to a non-negligible performance drag. + +------------------------------------------------ +A conceptual overview part 2: the nuts and bolts +------------------------------------------------ + +Part 2 goes into detail on the mechanisms :mod:`!asyncio` uses to manage +control flow. +This is where the magic happens. +You'll come away from this section knowing what ``await`` does behind the scenes +and how to make your own asynchronous operators. + +================================ +The inner workings of coroutines +================================ + +:mod:`!asyncio` leverages four components of Python to pass +around control. + +:meth:`coroutine.send(arg) ` is the method used to start or +resume a coroutine. +If the coroutine was paused and is now being resumed, the argument ``arg`` +will be sent in as the return value of the ``yield`` statement which originally +paused it. +If the coroutine is being used for the first time (as opposed to being resumed), +``arg`` must be ``None``. + +.. code-block:: + :linenos: + + class Rock: + def __await__(self): + value_sent_in = yield 7 + print(f"Rock.__await__ resuming with value: {value_sent_in}.") + return value_sent_in + + async def main(): + print("Beginning coroutine main().") + rock = Rock() + print("Awaiting rock...") + value_from_rock = await rock + print(f"Coroutine received value: {value_from_rock} from rock.") + return 23 + + coroutine = main() + intermediate_result = coroutine.send(None) + print(f"Coroutine paused and returned intermediate value: {intermediate_result}.") + + print(f"Resuming coroutine and sending in value: 42.") + try: + coroutine.send(42) + except StopIteration as e: + returned_value = e.value + print(f"Coroutine main() finished and provided value: {returned_value}.") + +:ref:`yield `, as usual, pauses execution and returns control +to the caller. +In the example above, the ``yield``, on line 3, is called by +``... = await rock`` on line 11. +More broadly speaking, ``await`` calls the :meth:`~object.__await__` method of +the given object. +``await`` also does one more very special thing: it propagates (or "passes +along") any ``yield``\ s it receives up the call chain. +In this case, that's back to ``... = coroutine.send(None)`` on line 16. + +The coroutine is resumed via the ``coroutine.send(42)`` call on line 21. +The coroutine picks back up from where it ``yield``\ ed (or paused) on line 3 +and executes the remaining statements in its body. +When a coroutine finishes, it raises a :exc:`StopIteration` exception with the +return value attached in the :attr:`~StopIteration.value` attribute. + +That snippet produces this output: + +.. code-block:: none + + Beginning coroutine main(). + Awaiting rock... + Coroutine paused and returned intermediate value: 7. + Resuming coroutine and sending in value: 42. + Rock.__await__ resuming with value: 42. + Coroutine received value: 42 from rock. + Coroutine main() finished and provided value: 23. + +It's worth pausing for a moment here and making sure you followed the various +ways that control flow and values were passed. A lot of important ideas were +covered and it's worth ensuring your understanding is firm. + +The only way to yield (or effectively cede control) from a coroutine is to +``await`` an object that ``yield``\ s in its ``__await__`` method. +That might sound odd to you. You might be thinking: + + 1. What about a ``yield`` directly within the coroutine function? The + coroutine function becomes an + :ref:`async generator function `, a + different beast entirely. + + 2. What about a :ref:`yield from ` within the coroutine function to a (plain) + generator? + That causes the error: ``SyntaxError: yield from not allowed in a coroutine.`` + This was intentionally designed for the sake of simplicity -- mandating only + one way of using coroutines. + Despite that, ``yield from`` and ``await`` effectively do the same thing. + Initially ``yield`` was barred as well, but was re-accepted to allow for + async generators. + +======= +Futures +======= + +A :ref:`future ` is an object meant to represent a +computation's status and result. +The term is a nod to the idea of something still to come or not yet happened, +and the object is a way to keep an eye on that something. + +A future has a few important attributes. One is its state, which can be either +"pending", "cancelled", or "done". +Another is its result, which is set when the state transitions to done. +Unlike a coroutine, a future does not represent the actual computation to be +done; instead, it represents the status and result of that computation, kind of +like a status light (red, yellow, or green) or indicator. + +:class:`asyncio.Task` subclasses :class:`asyncio.Future` in order to gain +these various capabilities. +The prior section said tasks store a list of callbacks, which wasn't entirely +accurate. +It's actually the ``Future`` class that implements this logic, which ``Task`` +inherits. + +Futures may also be used directly (not via tasks). +Tasks mark themselves as done when their coroutine is complete. +Futures are much more versatile and will be marked as done when you say so. +In this way, they're the flexible interface for you to make your own conditions +for waiting and resuming. + +======================== +A homemade asyncio.sleep +======================== + +We'll go through an example of how you could leverage a future to create your +own variant of asynchronous sleep (``async_sleep``) which mimics +:func:`asyncio.sleep`. + +This snippet registers a few tasks with the event loop and then awaits the task +created by ``asyncio.create_task``, which wraps the ``async_sleep(3)`` coroutine. +We want that task to finish only after three seconds have elapsed, but without +preventing other tasks from running. + +:: + + async def other_work(): + print("I like work. Work work.") + + async def main(): + # Add a few other tasks to the event loop, so there's something + # to do while asynchronously sleeping. + work_tasks = [ + asyncio.create_task(other_work()), + asyncio.create_task(other_work()), + asyncio.create_task(other_work()) + ] + print( + "Beginning asynchronous sleep at time: " + f"{datetime.datetime.now().strftime("%H:%M:%S")}." + ) + await asyncio.create_task(async_sleep(3)) + print( + "Done asynchronous sleep at time: " + f"{datetime.datetime.now().strftime("%H:%M:%S")}." + ) + # asyncio.gather effectively awaits each task in the collection. + await asyncio.gather(*work_tasks) + + +Below, we use a future to enable custom control over when that task will be +marked as done. +If :meth:`future.set_result() ` (the method +responsible for marking that future as done) is never called, then this task +will never finish. +We've also enlisted the help of another task, which we'll see in a moment, that +will monitor how much time has elapsed and, accordingly, call +``future.set_result()``. + +:: + + async def async_sleep(seconds: float): + future = asyncio.Future() + time_to_wake = time.time() + seconds + # Add the watcher-task to the event loop. + watcher_task = asyncio.create_task(_sleep_watcher(future, time_to_wake)) + # Block until the future is marked as done. + await future + +Below, we use a rather bare ``YieldToEventLoop()`` object to ``yield`` +from its ``__await__`` method, ceding control to the event loop. +This is effectively the same as calling ``asyncio.sleep(0)``, but this approach +offers more clarity, not to mention it's somewhat cheating to use +``asyncio.sleep`` when showcasing how to implement it! + +As usual, the event loop cycles through its tasks, giving them control +and receiving control back when they pause or finish. +The ``watcher_task``, which runs the coroutine ``_sleep_watcher(...)``, will +be invoked once per full cycle of the event loop. +On each resumption, it'll check the time and if not enough has elapsed, then +it'll pause once again and hand control back to the event loop. +Once enough time has elapsed, ``_sleep_watcher(...)`` +marks the future as done and completes by exiting its +infinite ``while`` loop. +Given this helper task is only invoked once per cycle of the event loop, +you'd be correct to note that this asynchronous sleep will sleep *at least* +three seconds, rather than exactly three seconds. +Note this is also true of ``asyncio.sleep``. + +:: + + class YieldToEventLoop: + def __await__(self): + yield + + async def _sleep_watcher(future, time_to_wake): + while True: + if time.time() >= time_to_wake: + # This marks the future as done. + future.set_result(None) + break + else: + await YieldToEventLoop() + +Here is the full program's output: + +.. code-block:: none + + $ python custom-async-sleep.py + Beginning asynchronous sleep at time: 14:52:22. + I like work. Work work. + I like work. Work work. + I like work. Work work. + Done asynchronous sleep at time: 14:52:25. + +You might feel this implementation of asynchronous sleep was unnecessarily +convoluted. +And, well, it was. +The example was meant to showcase the versatility of futures with a simple +example that could be mimicked for more complex needs. +For reference, you could implement it without futures, like so:: + + async def simpler_async_sleep(seconds): + time_to_wake = time.time() + seconds + while True: + if time.time() >= time_to_wake: + return + else: + await YieldToEventLoop() + +But that's all for now. Hopefully you're ready to more confidently dive into +some async programming or check out advanced topics in the +:mod:`rest of the documentation `. diff --git a/Doc/howto/abi3t-migration.rst b/Doc/howto/abi3t-migration.rst new file mode 100644 index 000000000000000..ed7a324c4af6f0a --- /dev/null +++ b/Doc/howto/abi3t-migration.rst @@ -0,0 +1,614 @@ +.. highlight:: c + +.. _abi3t-migration-howto: + +****************************************************** +Migrating to Stable ABI for free threading (``abi3t``) +****************************************************** + +Starting with the 3.15 release, CPython supports a variant of the Stable ABI +that supports :term:`free-threaded ` Python: +Stable ABI for Free-Threaded Builds, or ``abi3t`` for short. +This document describes how to adapt C API extensions to support free threading. + +Why do this +=========== + +The typical reason to use Stable ABI is to reduce the number of artifacts that +you need to build and distribute for each version of your library. + +Without the Stable ABI, you must build a separate shared library, and typically +a *wheel* distribution, for each feature version of CPython you wish +to support. +For example, each tag in the following table represents a separate +library/wheel: + ++-----------------+-----------------------+------------------------+ +| CPython version | Non-free-threaded | Free-threaded | ++=================+=======================+========================+ +| 3.12 | ``cpython-312`` | --- | ++-----------------+-----------------------+------------------------+ +| 3.13 | ``cpython-313`` | ``cpython-313t`` | ++-----------------+-----------------------+------------------------+ +| 3.14 | ``cpython-314`` | ``cpython-314t`` | ++-----------------+-----------------------+------------------------+ +| 3.15 | ``cpython-315`` | ``cpython-315t`` | ++-----------------+-----------------------+------------------------+ +| 3.16 | ``cpython-316`` | ``cpython-316t`` | ++-----------------+-----------------------+------------------------+ +| Later versions | :samp:`cpython-3{XX}` | :samp:`cpython-3{XX}t` | ++-----------------+-----------------------+------------------------+ + +That's a lot of builds, especially when multiplied by the number +of supported platforms. + +With the Stable ABI (``abi3``, introduced in CPython 3.2), a single extension +(per platform) can cover all *non-free-threaded* builds of CPython: + ++-----------------+-------------------+------------------------+ +| CPython version | Non-free-threaded | Free-threaded | ++=================+===================+========================+ +| 3.12 | ``abi3`` | --- | ++-----------------+ +------------------------+ +| 3.13 | | ``cpython-313t`` | ++-----------------+ +------------------------+ +| 3.14 | | ``cpython-314t`` | ++-----------------+ +------------------------+ +| 3.15 | | ``cpython-315t`` | ++-----------------+ +------------------------+ +| 3.16 | | ``cpython-316t`` | ++-----------------+ +------------------------+ +| Later versions | | :samp:`cpython-3{XX}t` | ++-----------------+-------------------+------------------------+ + +The Stable ABI for free-threaded builds (``abi3t``), introduced in +CPython 3.15, does the same for free-threaded builds. +And it's compatible with non-free-threaded ones as well: + ++-----------------+-------------------+------------------+ +| CPython version | Non-free-threaded | Free-threaded | ++=================+===================+==================+ +| 3.12 | ``abi3`` * | --- | ++-----------------+ +------------------+ +| 3.13 | | ``cpython-313t`` | ++-----------------+ +------------------+ +| 3.14 | | ``cpython-314t`` | ++-----------------+-------------------+------------------+ +| 3.15 | ``abi3t`` | ++-----------------+ + +| 3.16 | | ++-----------------+ + +| Later versions | | ++-----------------+-------------------+------------------+ + +\* (As above, the ``abi3`` extension is compatible with all non-free-threaded +builds; even the 3.15+ ones that this table "attributes" to ``abi3t``.) + +Why *not* do this +----------------- + +There are two main downsides to Stable ABI. + +First, you extension may become slower, since Stable ABI prioritizes +compatibility over performance. +The difference is usually not noticeable, and often can be mitigated by +using the same source to build both a Stable ABI build and a few +version-specific ones for "tier 1" CPython versions. + +Second, not all of the C API is available. +Extensions need to be ported to build for Stable ABI, which may be difficult +or, in rare cases, impossible. + +Specifically, ``abi3t`` requires APIs added in CPython 3.15. +If you want to build your extension for older versions of CPython from the +same source, you have two main options: + +- Use preprocessor conditionals. + + When following this guide, use ``#ifdef Py_TARGET_ABI3T`` blocks whenever + you are told to do a change that breaks the build on CPython versions you + care about. Keep the pre-existing code in ``#else`` blocks. + + For hand-written C extensions, this approach is reasonable down to + CPython 3.12, due to additions introduced in :pep:`697`. + Keeping compatibility with 3.11 and below may be worth it for code + generators (for example, Cython). + +- Do not port to ``abi3t``, and continue building separate extensions for + each version of CPython, until you can drop support for the older versions. + + This is a valid approach. Not all extensions need to switch to ``abi3t`` + right now. + + +Prerequisites +============= + +This guide assumes that you have an extension written directly in C (or C++), +which you want to port to ``abi3t``. + +If your extenstion uses a code generator (like Cython) or language binding +(like PyO3), it's best to wait until that tool has support for ``abi3t``. +If you maintain such a tool, you might be able to adapt the instructions +here for your tool. + +Non-free-threaded Stable ABI +---------------------------- + +Your extension should support the Stable ABI (``abi3t``). +If not, either port it first, or follow this guide but be prepared to fix +issues it does not mention. + +Free-threading support +---------------------- + +While it's technically not a hard prerequisite, you will most likely want to +prepare your extension for free threading before you port it to ``abi3t``. +See :ref:`freethreading-extensions-howto` for instructions. + +.. seealso:: + + `Porting Extension Modules to Support Free-Threading + `__: + A community-maintained porting guide for extension authors. + +Isolating extension modules +--------------------------- + +Your module should use :ref:`multi-phase initialization `, +and it should either be isolated or limit itself to be loaded at most once +per process. +If it is not your case, follow :ref:`isolating-extensions-howto` first. +(See the :ref:`opt-out section ` for a shortcut.) + +Avoiding variable-sized types +----------------------------- + +If your extension defines variable-sized types (using :c:macro:`Py_tp_itemsize` +or :c:member:`PyTypeObject.tp_itemsize`), it cannot be ported to +``abi3t`` 3.15. + + +Setting up the build +==================== + +If you use a build tool (such as setuptools, meson-python, scikit-build-core), +search its documentation for a way to select ``abi3t``. +At the time of writing, not all of them have this; but if your tool does, +use it. +You may want to verify that it set the right flag by temporarily adding the +following just after ``#include ``:: + + #if Py_TARGET_ABI3T+0 <= 0x30f0000 + #error "abi3t define is not set!" + #endif + +This should result in a different error than "``abt3t`` define is not set". + +.. note:: + + If your build tool doesn't support ``abi3t`` yet, set the following macro + before including ``Python.h``:: + + #define Py_TARGET_ABI3T 0x30f0000 + + or specify it as a compiler flag, for example:: + + -DPy_TARGET_ABI3T=0x30f0000 + + Once your extension builds with this setting, it will be compatible with + CPython 3.15 and above. + + If you set this macro manually, you will later need to name and tag the + resulting extension manually as well. + This is covered in :ref:`abi3t-migration-tagging` below. + +This guide will ask you to make a series of changes. +After each one, verify that your extension still builds in the original +(non-``abi3t``) configuration, and ideally run tests on all Python +versions you support. +This will ensure that nothing breaks as you are porting. + + +Module export hook +================== + +Unless you've done this step already, your extension module defines a +:ref:`module initialization function ` +named :samp:`PyInit_{}`. +You will need to port it to a :ref:`module export hook `, +:samp:`PyModExport_{}`, a feature added in CPython 3.15 in +:pep:`793`. + +Your existing init function should look like this (with your own names +for ```` and ````): + +.. code-block:: + :class: bad + + PyMODINIT_FUNC + PyInit_(void) + { + return PyModuleDef_Init(&); + } + +If there is some code before the ``return``, move it to +a :c:macro:`Py_mod_create` or :c:macro:`Py_mod_exec` slot function. +See the :ref:`PyInit documentation ` for related information. + +The function references a ``PyModuleDef`` object (```` in the code +above). +Its definition should be similar to the following, with different values +and perhaps some fields unnnamed or left out: + +.. code-block:: + :class: bad + + static PyModuleDef = { + PyModuleDef_HEAD_INIT, + .m_name = "my_module", + .m_doc = "my docstring", + .m_size = sizeof(my_state_struct), + .m_methods = my_methods, + .m_slots = my_slots, + .m_traverse = my_traverse, + .m_clear = my_clear, + .m_free = my_free, + }; + +Remove this definition and the ``PyInit`` function (or put them in +an ``#ifndef Py_TARGET_ABI3T`` block, to retain backwards compatibility), +and replace them with the following: + +.. code-block:: + :class: good + + PyABIInfo_VAR(abi_info); + + static PySlot my_slot_array[] = { + PySlot_STATIC_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_name, "my_module"), + PySlot_STATIC_DATA(Py_mod_doc, "my docstring"), + PySlot_SIZE(Py_mod_state_size, sizeof(my_state_struct)), + PySlot_STATIC_DATA(Py_mod_methods, my_methods), + PySlot_STATIC_DATA(Py_mod_slots, my_slots), + PySlot_FUNC(Py_mod_state_traverse, my_traverse), + PySlot_FUNC(Py_mod_state_clear, my_clear), + PySlot_FUNC(Py_mod_state_free, my_free), + PySlot_END + }; + + PyMODEXPORT_FUNC + PyModExport_(void) + { + return my_slot_array; + } + +Leave out any fields that were missing (except the new :c:macro:`Py_mod_abi`), +and substitute your own values. + +See the :c:type:`PySlot` and :c:ref:`export hook ` +documentation for details on this API. + +Associated ``PyModuleDef`` +-------------------------- + +Since the new API does not use a :c:type:`!PyModuleDef` structure, a definition +will not be associated with the resulting module. +This changes the behavior of the following functions: + +- :c:func:`PyModule_GetDef` +- :c:func:`PyType_GetModuleByDef` + +Check your code for these. +If you do not use them, you can skip this section. + +These functions are typically used for two purposes: + +1. To get the definition the module was created with. + This is no longer possible using the new API. + Modules no longer keep a reference to the definition, so you will need to + figure out a different way to pass the relevant data around. + +.. _abi3t-migration-module-token: + +2. To check if a given module object is “yours”. + This use case is now served by :ref:`module tokens ` -- + opaque pointers that identify a module. + To use a token, declare (or reuse) a unique static variable, for example: + + .. code-block:: + :class: good + + static char my_token; + + and add a pointer to it in a new entry to your module's ``PySlot`` array: + + .. code-block:: + :class: good + :emphasize-lines: 3 + + static PySlot my_slot_array[] = { + ... + PySlot_STATIC_DATA(Py_mod_token, &my_token), + PySlot_END + } + + Then, switch from :c:func:`PyModule_GetDef` calls such as: + + .. code-block:: + :class: bad + + PyModuleDef *def = PyModule_GetDef(module); + + to :c:func:`PyModule_GetToken` (which uses an output argument and may fail + with an exception): + + .. code-block:: + :class: good + + void *token; + if (PyModule_GetToken(module, &token) < 0) { + /* handle error */ + } + + and from :c:func:`PyType_GetModuleByDef` calls such as: + + .. code-block:: + :class: bad + + PyObject *module = PyType_GetModuleByDef(type, my_def); + /* handle error; use module */ + + to :c:func:`PyType_GetModuleByToken` (which returns a strong reference): + + .. code-block:: + :class: good + + PyObject *module = PyType_GetModuleByToken(type, my_token); + /* handle error; use module */ + Py_XDECREF(module); + +``PyObject`` opaqueness +======================= + +The :c:type:`PyObject` and :c:type:`PyVarObject` structures are opaque +in ``abi3t``. + +Accessing their members is prohibited. +If you do this, switch to getter/setter functions mentioned in +their documentation: + +- :c:member:`PyObject.ob_type` +- :c:member:`PyObject.ob_refcnt` +- :c:member:`PyVarObject.ob_size` + +Also, the *size* of the :c:type:`PyObject` structures is +unknown to the compiler. +It can -- and *does* -- change between different CPython builds. + +.. note:: + + While the size is available at runtime (for example as + ``sys.getsizeof(object())`` in Python code), you should resist the + temptation to calculate pointer offsets from it. + The object memory layout is subject to change in future + ``abi3t`` implementations. + + +Custom type definitions +----------------------- + +Since :c:type:`!PyObject` is opaque, the traditional way of defining +custom types no longer works: + +.. code-block:: + :class: bad + + typedef struct { + PyObject_HEAD // expands to `PyObject ob_base;` which has unknown size + + int my_data; + } CustomObject; + + static PyType_Spec CustomType_spec = { + ... + .basicsize = sizeof(CustomObject), + ... + }; + +Most likely, all your class definitions, *and* all code that accesses +your classes' data, will need to be rewritten. +This will probably be the biggest change you need to support ``abi3t``. + +For each such type, instead of defining a ``struct`` for the entire instance, +define one with only the “additional” fields -- ones specific to your class, +not its superclasses: + +.. code-block:: + :class: good + + typedef struct { + int my_data; + } CustomObjectData; + +Change the name. +Almost all code that uses the struct will need to change +(notably, pointers to the new structure cannot be cast to/from ``PyObject*``), +and changing the name will highlight the usages as compiler errors. +(If you use ``typeof``, C++ ``auto``, or similar ways to avoid +typing the type name, this won't work. Be extra careful, and consider running +tools to detect undefined behavior.) + +Then, to create the class, use *negative* ``basicsize`` to indicate +“extra” storage space rather than *total* instance size: + +.. code-block:: + :class: good + + static PyType_Spec CustomType_spec = { + ... + .basicsize = -sizeof(CustomObjectData), /* note the minus sign */ + ... + }; + +If you use :c:macro:`Py_tp_members`, set the :c:macro:`Py_RELATIVE_OFFSET` +flag on each member and specify the :c:member:`~PyMemberDef.offset` +relative to your new struct. + + +Custom type data access +----------------------- + +Then comes the hard part: in all code that needs to access this struct, +you will need an additional :c:func:`PyObject_GetTypeData` call to +retrieve a ``CustomObjectData *`` pointer from ``PyObject *``: + +.. code-block:: + :class: good + + PyObject *obj = ...; + CustomObjectData *data = PyObject_GetTypeData(obj, cls); + +Note that this call requires the *type object* for your class (``cls``). + +If your class is not subclassable (that is, it does not use the +:c:macro:`Py_TPFLAGS_BASETYPE` flag), ``cls`` will be ``Py_TYPE(obj)``. +Otherwise, **DO NOT USE** ``Py_TYPE`` with :c:func:`!PyObject_GetTypeData`: +it might return memory reserved to an unrelated subclass! +For example, if a user makes a subclass like this: + +.. code-block:: python + + class Sub(YourCustomClass): + __slots__ = ('a', 'b') + +then ``Py_TYPE(obj)`` is ``YourCustomClass``, and the underlying memory may +look like this: + +.. code-block:: text + + ╭─ PyObject *obj + │ ╭─ the pointer you want + │ │ ╭─ PyObject_GetTypeData(obj, Py_TYPE(obj)) + ▼ ▼ ▼ + ┌──────────┬───┬────────────────┬───┬─────────────┬───┬─────────────┐ + │ PyObject │...│ CustomTypeData │...│ PyObject *a │...│ PyObject *b │ + └──────────┴───┴────────────────┴───┴─────────────┴───┴─────────────┘ + +(Ellipses indicate possible padding. +Note that this memory layout is not guaranteed: future versions of Python may +add different padding or even switch the order of the structures.) + +There are two main ways to get the right class: + +- In instance methods, your implementation may use the :c:type:`PyCMethod` + signature (and the :c:macro:`METH_METHOD` bit in + :c:member:`PyMethodDef.ml_flags`), + and get the class as the ``defining_class`` argument. +- Otherwise, give your class a unique static token using the + :c:macro:`Py_tp_token` slot, and use: + + .. code-block:: + :class: good + + PyTypeObject cls; + if (PyType_GetBaseByToken(Py_TYPE(obj), my_tp_token, &cls) < 0) { + /* handle error */ + } + CustomObjectData *data = PyObject_GetTypeData(obj, cls); + + Type tokens work similarly to module tokens covered :ref:`earlier in this + guide `. + + + +Avoid build-time conditionals +============================= + +Check your code for API that identifies the version of Python used to +*build* your extension. +This no longer corresponds to the Python your extension runs on, so code +that uses this information often needs changing. +The macros to check for are: + +- :c:macro:`PY_VERSION_HEX`, :c:macro:`PY_MAJOR_VERSION`, + :c:macro:`PY_MINOR_VERSION`: + + - to get the run-time version, use :c:data:`Py_Version`; + - to determine what C API is available, use :c:macro:`Py_TARGET_ABI3T`. + This macro is set to the minimum supported version. + +- :c:macro:`Py_GIL_DISABLED`: under ``abi3t``, this macro is always defined. + Code that works with free-threaded Python *should* also work with + the GIL enabled (since the GIL can be enabled at run time), + and usually *does* (unless it, for some reason, requires more than one + :term:`attached thread state ` at one time). + + +Further code changes +==================== + +If you are still left with compiler errors or warnings, find a way to fix them. +Alas, this guide is limited, and cannot cover all possible code +changes extensions may need. + +If you find a problem that other extension authors might run into, +consider :ref:`reporting an issue ` (or sending +a pull request) for this guide. + +It is possible your issue cannot be fixed for the current version of ``abi3t``. +In that case, reporting it may help it get prioritized for the next version +of CPython. + + +.. _abi3t-migration-tagging: + +Tagging and distribution +======================== + +If you are using a build tool with ``abi3t`` support, your extension is ready, +but you might want to check that it was built correctly. + +Extensions built with ``abi3t`` should have the following extension: + +- On Windows: ``.pyd`` (like any other extension); +- Linux, macOS, and other systems that use the ``.so`` suffix: ``.abi3t.so`` + (**not** ``.cpython-315t.so`` or ``.abi3.so``). + Note that both free-threaded and non-free-threaded builds will + load ``.abi3t.so`` extensions; +- Other systems: consult your distributor, and perhaps update this guide. + +If you distribute the extension as a *wheel*, use the following tags: + +* Python tag: :samp:`cp3{XX}`, where *XX* is the minimum Python version + the extension is built for. + (For example, ``cp315`` if you set ``Py_TARGET_ABI3T`` to ``0x30f0000``. + See :ref:`abi3-compiling` for more values.) +* ABI tag: ``abi3.abi3t``. This is a *compressed tag set* that indicates + support for both non-free-threaded and free-threaded builds. + +For example, the wheel filename may look like this: + +.. code-block:: text + + myproject-1.0-cp315-abi3.abi3t-macosx_11_0_arm64.whl + +.. seealso:: `Platform Compatibility Tags `__ in the PyPA package distribution metadata. + +If the filename or tags are incorrect, fix them. + + +Testing +======= + +Note that when you build an extension compatible with multiple versions of +CPython, you should always *test* it with each version it supports (for +example, 3.15, 3.16, and so on). +Stable ABI only guarantees *ABI* compatibility; there may also be behavior +changes -- both intentional ones (covered by :pep:`387`) and bugs. + +Be sure to run tests on both free-threaded and non-free-threaded builds +of CPython. + +If they pass, congratulations! You have an ``abi3t`` extension. diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index d7deb6c6bc1768f..fcc4fe838c783ba 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -4,8 +4,6 @@ Annotations Best Practices ************************** -:author: Larry Hastings - .. topic:: Abstract This document is designed to encapsulate the best practices @@ -154,7 +152,7 @@ on an arbitrary object ``o``: as the ``globals``, and ``dict(vars(o))`` as the ``locals``, when calling :func:`eval`. * If ``o`` is a wrapped callable using :func:`functools.update_wrapper`, - :func:`functools.wraps`, or :func:`functools.partial`, iteratively + :deco:`functools.wraps`, or :func:`functools.partial`, iteratively unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as appropriate, until you have found the root unwrapped function. * If ``o`` is a callable (but not a class), use diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 902c50de00803c5..0cb8c5cc3ebd369 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -4,8 +4,6 @@ Argparse Tutorial ***************** -:author: Tshepang Mbambo - .. currentmodule:: argparse This tutorial is intended to be a gentle introduction to :mod:`argparse`, the diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst index 816639552d7cd6d..e5f85e0110321c8 100644 --- a/Doc/howto/curses.rst +++ b/Doc/howto/curses.rst @@ -6,9 +6,6 @@ .. currentmodule:: curses -:Author: A.M. Kuchling, Eric S. Raymond -:Release: 2.04 - .. topic:: Abstract diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index f6c3e473f1c36d3..7233898860ada14 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -4,9 +4,6 @@ Descriptor Guide ================ -:Author: Raymond Hettinger -:Contact: - .. Contents:: @@ -28,7 +25,7 @@ This guide has four major sections: 4) The last section has pure Python equivalents for built-in descriptors that are written in C. Read this if you're curious about how functions turn into bound methods or about the implementation of common tools like - :func:`classmethod`, :func:`staticmethod`, :func:`property`, and + :deco:`classmethod`, :deco:`staticmethod`, :deco:`property`, and :term:`__slots__`. @@ -317,8 +314,8 @@ Descriptors invert that relationship and allow the data being looked-up to have a say in the matter. Descriptors are used throughout the language. It is how functions turn into -bound methods. Common tools like :func:`classmethod`, :func:`staticmethod`, -:func:`property`, and :func:`functools.cached_property` are all implemented as +bound methods. Common tools like :deco:`classmethod`, :deco:`staticmethod`, +:deco:`property`, and :deco:`functools.cached_property` are all implemented as descriptors. @@ -420,7 +417,7 @@ Here are three practical data validation utilities: def validate(self, value): if not isinstance(value, str): - raise TypeError(f'Expected {value!r} to be an str') + raise TypeError(f'Expected {value!r} to be a str') if self.minsize is not None and len(value) < self.minsize: raise ValueError( f'Expected {value!r} to be no smaller than {self.minsize!r}' @@ -594,7 +591,7 @@ a pure Python equivalent: def object_getattribute(obj, name): "Emulate PyObject_GenericGetAttr() in Objects/object.c" - null = object() + null = sentinel('null') objtype = type(obj) cls_var = find_name_in_mro(objtype, name, null) descr_get = getattr(type(cls_var), '__get__', null) @@ -1326,7 +1323,7 @@ example calls are unexciting: 30 Using the non-data descriptor protocol, a pure Python version of -:func:`staticmethod` would look like this: +:deco:`staticmethod` would look like this: .. testcode:: @@ -1466,7 +1463,7 @@ Now a new dictionary of unique keys can be constructed like this: {'a': None, 'b': None, 'r': None, 'c': None, 'd': None} Using the non-data descriptor protocol, a pure Python version of -:func:`classmethod` would look like this: +:deco:`classmethod` would look like this: .. testcode:: @@ -1604,7 +1601,7 @@ matters when a large number of instances are going to be created. 4. Improves speed. Reading instance variables is 35% faster with ``__slots__`` (as measured with Python 3.10 on an Apple M1 processor). -5. Blocks tools like :func:`functools.cached_property` which require an +5. Blocks tools like :deco:`functools.cached_property` which require an instance dictionary to function correctly: .. testcode:: @@ -1635,12 +1632,12 @@ by member descriptors: .. testcode:: - null = object() + null = sentinel('null') class Member: def __init__(self, name, clsname, offset): - 'Emulate PyMemberDef in Include/structmember.h' + 'Emulate PyMemberDef in Include/descrobject.h' # Also see descr_new() in Objects/descrobject.c self.name = name self.clsname = clsname diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 6441b7aed1eda8c..adb9dc9a4879eb5 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -105,8 +105,8 @@ The complete :class:`!Weekday` enum now looks like this:: Now we can find out what today is! Observe:: - >>> from datetime import date - >>> Weekday.from_date(date.today()) # doctest: +SKIP + >>> import datetime as dt + >>> Weekday.from_date(dt.date.today()) # doctest: +SKIP Of course, if you're reading this on some other day, you'll see that day instead. @@ -256,7 +256,7 @@ Ensuring unique enumeration values ---------------------------------- By default, enumerations allow multiple names as aliases for the same value. -When this behavior isn't desired, you can use the :func:`unique` decorator:: +When this behavior isn't desired, you can use the :deco:`unique` decorator:: >>> from enum import Enum, unique >>> @unique @@ -371,7 +371,7 @@ Equality comparisons are defined though:: >>> Color.BLUE == Color.BLUE True -Comparisons against non-enumeration values will always compare not equal +Equality comparisons against non-enumeration values will always return ``False`` (again, :class:`IntEnum` was explicitly designed to behave differently, see below):: @@ -509,7 +509,7 @@ to use the standard :func:`repr`. .. note:: - Adding :func:`~dataclasses.dataclass` decorator to :class:`Enum` + Adding :deco:`~dataclasses.dataclass` decorator to :class:`Enum` and its subclasses is not supported. It will not raise any errors, but it will produce very strange results at runtime, such as members being equal to each other:: @@ -965,75 +965,16 @@ want one of them to be the value:: Finer Points -^^^^^^^^^^^^ - -Supported ``__dunder__`` names -"""""""""""""""""""""""""""""" - -:attr:`~enum.EnumType.__members__` is a read-only ordered mapping of ``member_name``:``member`` -items. It is only available on the class. - -:meth:`~object.__new__`, if specified, must create and return the enum members; it is -also a very good idea to set the member's :attr:`~Enum._value_` appropriately. Once -all the members are created it is no longer used. - - -Supported ``_sunder_`` names -"""""""""""""""""""""""""""" +------------ -- :attr:`~Enum._name_` -- name of the member -- :attr:`~Enum._value_` -- value of the member; can be set in ``__new__`` -- :meth:`~Enum._missing_` -- a lookup function used when a value is not found; - may be overridden -- :attr:`~Enum._ignore_` -- a list of names, either as a :class:`list` or a - :class:`str`, that will not be transformed into members, and will be removed - from the final class -- :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for - an enum member; may be overridden -- :meth:`~EnumType._add_alias_` -- adds a new name as an alias to an existing - member. -- :meth:`~EnumType._add_value_alias_` -- adds a new value as an alias to an - existing member. See `MultiValueEnum`_ for an example. +Supported ``__dunder__`` and ``_sunder_`` names +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - .. note:: - - For standard :class:`Enum` classes the next value chosen is the highest - value seen incremented by one. - - For :class:`Flag` classes the next value chosen will be the next highest - power-of-two. - - .. versionchanged:: 3.13 - Prior versions would use the last seen value instead of the highest value. - -.. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` -.. versionadded:: 3.7 ``_ignore_`` -.. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_`` - -To help keep Python 2 / Python 3 code in sync an :attr:`~Enum._order_` attribute can -be provided. It will be checked against the actual order of the enumeration -and raise an error if the two do not match:: - - >>> class Color(Enum): - ... _order_ = 'RED GREEN BLUE' - ... RED = 1 - ... BLUE = 3 - ... GREEN = 2 - ... - Traceback (most recent call last): - ... - TypeError: member order does not match _order_: - ['RED', 'BLUE', 'GREEN'] - ['RED', 'GREEN', 'BLUE'] - -.. note:: - - In Python 2 code the :attr:`~Enum._order_` attribute is necessary as definition - order is lost before it can be recorded. +The supported ``__dunder__`` and ``_sunder_`` names can be found in the :ref:`Enum API documentation `. _Private__names -""""""""""""""" +^^^^^^^^^^^^^^^ :ref:`Private names ` are not converted to enum members, but remain normal attributes. @@ -1042,7 +983,7 @@ but remain normal attributes. ``Enum`` member type -"""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^ Enum members are instances of their enum class, and are normally accessed as ``EnumClass.member``. In certain situations, such as writing custom enum @@ -1055,7 +996,7 @@ recommended. Creating members that are mixed with other data types -""""""""""""""""""""""""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When subclassing other data types, such as :class:`int` or :class:`str`, with an :class:`Enum`, all values after the ``=`` are passed to that data type's @@ -1069,7 +1010,7 @@ constructor. For example:: Boolean value of ``Enum`` classes and members -""""""""""""""""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Enum classes that are mixed with non-:class:`Enum` types (such as :class:`int`, :class:`str`, etc.) are evaluated according to the mixed-in @@ -1084,7 +1025,7 @@ Plain :class:`Enum` classes always evaluate as :data:`True`. ``Enum`` classes with methods -""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you give your enum subclass extra methods, like the `Planet`_ class below, those methods will show up in a :func:`dir` of the member, @@ -1097,7 +1038,7 @@ but not of the class:: Combining members of ``Flag`` -""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Iterating over a combination of :class:`Flag` members will only return the members that are comprised of a single bit:: @@ -1117,7 +1058,7 @@ are comprised of a single bit:: ``Flag`` and ``IntFlag`` minutia -"""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Using the following snippet for our examples:: @@ -1478,6 +1419,7 @@ alias:: behaviors as well as disallowing aliases. If the only desired change is disallowing aliases, the :func:`unique` decorator can be used instead. +.. _multi-value-enum: MultiValueEnum ^^^^^^^^^^^^^^^^^ @@ -1538,8 +1480,8 @@ TimePeriod An example to show the :attr:`~Enum._ignore_` attribute in use:: - >>> from datetime import timedelta - >>> class Period(timedelta, Enum): + >>> import datetime as dt + >>> class Period(dt.timedelta, Enum): ... "different lengths of time" ... _ignore_ = 'Period i' ... Period = vars() diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 577e283bb9cb4c6..ad0578df0a27029 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -45,9 +45,12 @@ single-phase initialization. Multi-Phase Initialization .......................... -Extensions that use multi-phase initialization (i.e., -:c:func:`PyModuleDef_Init`) should add a :c:data:`Py_mod_gil` slot in the -module definition. If your extension supports older versions of CPython, +Extensions that use :ref:`multi-phase initialization ` +(functions like :c:func:`PyModuleDef_Init`, +:c:func:`PyModExport_* ` export hook, +:c:func:`PyModule_FromSlotsAndSpec`) should add a +:c:data:`Py_mod_gil` slot in the module definition. +If your extension supports older versions of CPython, you should guard the slot with a :c:data:`PY_VERSION_HEX` check. :: @@ -60,18 +63,12 @@ you should guard the slot with a :c:data:`PY_VERSION_HEX` check. {0, NULL} }; - static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - .m_slots = module_slots, - ... - }; - Single-Phase Initialization ........................... -Extensions that use single-phase initialization (i.e., -:c:func:`PyModule_Create`) should call :c:func:`PyUnstable_Module_SetGIL` to +Extensions that use legacy :ref:`single-phase initialization ` +(that is, :c:func:`PyModule_Create`) should call :c:func:`PyUnstable_Module_SetGIL` to indicate that they support running with the GIL disabled. The function is only defined in the free-threaded build, so you should guard the call with ``#ifdef Py_GIL_DISABLED`` to avoid compilation errors in the regular build. @@ -173,9 +170,9 @@ that return :term:`strong references `. +-----------------------------------+-----------------------------------+ | :c:func:`PyDict_Next` | none (see :ref:`PyDict_Next`) | +-----------------------------------+-----------------------------------+ -| :c:func:`PyWeakref_GetObject` | :c:func:`PyWeakref_GetRef` | +| :c:func:`!PyWeakref_GetObject` | :c:func:`PyWeakref_GetRef` | +-----------------------------------+-----------------------------------+ -| :c:func:`PyWeakref_GET_OBJECT` | :c:func:`PyWeakref_GetRef` | +| :c:func:`!PyWeakref_GET_OBJECT` | :c:func:`PyWeakref_GetRef` | +-----------------------------------+-----------------------------------+ | :c:func:`PyImport_AddModule` | :c:func:`PyImport_AddModuleRef` | +-----------------------------------+-----------------------------------+ @@ -203,7 +200,7 @@ Memory Allocation APIs Python's memory management C API provides functions in three different :ref:`allocation domains `: "raw", "mem", and "object". For thread-safety, the free-threaded build requires that only Python objects -are allocated using the object domain, and that all Python object are +are allocated using the object domain, and that all Python objects are allocated using that domain. This differs from the prior Python versions, where this was only a best practice and not a hard requirement. @@ -221,13 +218,15 @@ Thread State and GIL APIs Python provides a set of functions and macros to manage thread state and the GIL, such as: +* :c:func:`PyThreadState_Ensure`, :c:func:`PyThreadState_EnsureFromView`, + and :c:func:`PyThreadState_Release` * :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` * :c:func:`PyEval_SaveThread` and :c:func:`PyEval_RestoreThread` * :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` These functions should still be used in the free-threaded build to manage thread state even when the :term:`GIL` is disabled. For example, if you -create a thread outside of Python, you must call :c:func:`PyGILState_Ensure` +create a thread outside of Python, you must call :c:func:`PyThreadState_Ensure` before calling into the Python API to ensure that the thread has a valid Python thread state. @@ -344,12 +343,12 @@ This means you cannot rely on nested critical sections to lock multiple objects at once, as the inner critical section may suspend the outer ones. Instead, use :c:macro:`Py_BEGIN_CRITICAL_SECTION2` to lock two objects simultaneously. -Note that the locks described above are only :c:type:`!PyMutex` based locks. +Note that the locks described above are only :c:type:`PyMutex` based locks. The critical section implementation does not know about or affect other locking mechanisms that might be in use, like POSIX mutexes. Also note that while -blocking on any :c:type:`!PyMutex` causes the critical sections to be +blocking on any :c:type:`PyMutex` causes the critical sections to be suspended, only the mutexes that are part of the critical sections are -released. If :c:type:`!PyMutex` is used without a critical section, it will +released. If :c:type:`PyMutex` is used without a critical section, it will not be released and therefore does not get the same deadlock avoidance. Important Considerations @@ -387,6 +386,30 @@ Important Considerations internal extension state, standard mutexes or other synchronization primitives might be more appropriate. +.. _per-object-locks: + +Per-Object Locks (``ob_mutex``) +............................... + +In the free-threaded build, each Python object contains a :c:member:`~PyObject.ob_mutex` +field of type :c:type:`PyMutex`. This mutex is **reserved for use by the +critical section API** (:c:macro:`Py_BEGIN_CRITICAL_SECTION` / +:c:macro:`Py_END_CRITICAL_SECTION`). + +.. warning:: + + Do **not** lock ``ob_mutex`` directly with ``PyMutex_Lock(&obj->ob_mutex)``. + Mixing direct ``PyMutex_Lock`` calls with the critical section API on the + same mutex can cause deadlocks. + +Even if your own code never uses critical sections on a particular object type, +**CPython internals may use the critical section API on any Python object**. + +If your extension type needs its own lock, add a separate :c:type:`PyMutex` +field (or another synchronization primitive) to your object struct. +:c:type:`PyMutex` is very lightweight, so there is negligible cost to having +an additional one. + Building Extensions for the Free-Threaded Build =============================================== @@ -395,10 +418,9 @@ C API extensions need to be built specifically for the free-threaded build. The wheels, shared libraries, and binaries are indicated by a ``t`` suffix. * `pypa/manylinux `_ supports the - free-threaded build, with the ``t`` suffix, such as ``python3.13t``. -* `pypa/cibuildwheel `_ supports the - free-threaded build if you set - `CIBW_ENABLE to cpython-freethreading `_. + free-threaded build, with the ``t`` suffix, such as ``python3.14t``. +* `pypa/cibuildwheel `_ supports + building wheels for the free-threaded build of Python 3.14 and newer. Limited C API and Stable ABI ............................ diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst index 24069617c47ae18..53bea1db191d76f 100644 --- a/Doc/howto/free-threading-python.rst +++ b/Doc/howto/free-threading-python.rst @@ -11,9 +11,7 @@ available processing power by running threads in parallel on available CPU cores While not all software will benefit from this automatically, programs designed with threading in mind will run faster on multi-core hardware. -The free-threaded mode is working and continues to be improved, but -there is some additional overhead in single-threaded workloads compared -to the regular build. Additionally, third-party packages, in particular ones +Some third-party packages, in particular ones with an :term:`extension module`, may not be ready for use in a free-threaded build, and will re-enable the :term:`GIL`. @@ -101,60 +99,42 @@ This section describes known limitations of the free-threaded CPython build. Immortalization --------------- -The free-threaded build of the 3.13 release makes some objects :term:`immortal`. +In the free-threaded build, some objects are :term:`immortal`. Immortal objects are not deallocated and have reference counts that are never modified. This is done to avoid reference count contention that would prevent efficient multi-threaded scaling. -An object will be made immortal when a new thread is started for the first time -after the main thread is running. The following objects are immortalized: +As of the 3.14 release, immortalization is limited to: -* :ref:`function ` objects declared at the module level -* :ref:`method ` descriptors -* :ref:`code ` objects -* :term:`module` objects and their dictionaries -* :ref:`classes ` (type objects) - -Because immortal objects are never deallocated, applications that create many -objects of these types may see increased memory usage. This is expected to be -addressed in the 3.14 release. - -Additionally, numeric and string literals in the code as well as strings -returned by :func:`sys.intern` are also immortalized. This behavior is -expected to remain in the 3.14 free-threaded build. +* Code constants: numeric literals, string literals, and tuple literals + composed of other constants. +* Strings interned by :func:`sys.intern`. Frame objects ------------- -It is not safe to access :ref:`frame ` objects from other -threads and doing so may cause your program to crash . This means that -:func:`sys._current_frames` is generally not safe to use in a free-threaded -build. Functions like :func:`inspect.currentframe` and :func:`sys._getframe` -are generally safe as long as the resulting frame object is not passed to -another thread. +It is not safe to access :attr:`frame.f_locals` from a :ref:`frame ` +object if that frame is currently executing in another thread, and doing so may +crash the interpreter. + Iterators --------- -Sharing the same iterator object between multiple threads is generally not -safe and threads may see duplicate or missing elements when iterating or crash -the interpreter. +It is generally not thread-safe to access the same iterator object from +multiple threads concurrently, and threads may see duplicate or missing +elements. Single-threaded performance --------------------------- The free-threaded build has additional overhead when executing Python code -compared to the default GIL-enabled build. In 3.13, this overhead is about -40% on the `pyperformance `_ suite. -Programs that spend most of their time in C extensions or I/O will see -less of an impact. The largest impact is because the specializing adaptive -interpreter (:pep:`659`) is disabled in the free-threaded build. We expect -to re-enable it in a thread-safe way in the 3.14 release. This overhead is -expected to be reduced in upcoming Python release. We are aiming for an -overhead of 10% or less on the pyperformance suite compared to the default -GIL-enabled build. +compared to the default GIL-enabled build. The amount of overhead depends +on the workload and hardware. On the pyperformance benchmark suite, the +average overhead ranges from about 1% on macOS aarch64 to 8% on x86-64 Linux +systems. Behavioral changes @@ -185,3 +165,132 @@ to false. If the flag is true then the :class:`warnings.catch_warnings` context manager uses a context variable for warning filters. If the flag is false then :class:`~warnings.catch_warnings` modifies the global filters list, which is not thread-safe. See the :mod:`warnings` module for more details. + + +Increased memory usage +---------------------- + +The free-threaded build will typically use more memory compared to the default +build. There are multiple reasons for this, mostly due to design decisions. + + +All interned strings are immortal +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For modern Python versions (since version 2.3), interning a string (e.g. with +:func:`sys.intern`) does not cause it to become immortal. Instead, if the last +reference to that string disappears, it will be removed from the interned +string table. This is not the case for the free-threaded build and any interned +string will become immortal, surviving until interpreter shutdown. + + +Non-GC objects have a larger object header +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The free-threaded build uses a different :c:type:`PyObject` structure. Instead +of having the GC related information allocated before the :c:type:`PyObject` +structure, like in the default build, the GC related info is part of the normal +object header. For example, on the AMD64 platform, ``None`` uses 32 bytes on +the free-threaded build vs 16 bytes for the default build. GC objects (such as +dicts and lists) are the same size for both builds since the free-threaded +build does not use additional space for the GC info. + + +QSBR can delay freeing of memory +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In order to safely implement lock-free data structures, a safe memory +reclamation (SMR) scheme is used, known as quiescent state-based reclamation +(QSBR). This means that the memory backing data structures allowing lock-free +access will use QSBR, which defers the free operation, rather than immediately +freeing the memory. Two examples of these data structures are the list object +and the dictionary keys object. See ``InternalDocs/qsbr.md`` in the CPython +source tree for more details on how QSBR is implemented. Running +:func:`gc.collect` should cause all memory being held by QSBR to be actually +freed. Note that even when QSBR frees the memory, the underlying memory +allocator may not immediately return that memory to the OS and so the resident +set size (RSS) of the process might not decrease. + + +mimalloc allocator vs pymalloc +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The default build will normally use the "pymalloc" memory allocator for small +allocations (512 bytes or smaller). The free-threaded build does not use +pymalloc and allocates all Python objects using the "mimalloc" allocator. The +pymalloc allocator has the following properties that help keep memory usage +low: small per-allocated-block overhead, effective memory fragmentation +prevention, and quick return of free memory to the operating system. The +mimalloc allocator does quite well in these respects as well but can have some +more overhead. + +In the free-threaded build, mimalloc manages memory in a number of separate +heaps (currently four). For example, all GC supporting objects are allocated +from their own heap. Using separate heaps means that free memory in one heap +cannot be used for an allocation that uses another heap. Also, some heaps are +configured to use QSBR (quiescent-state based reclamation) when freeing the +memory that backs up the heap (known as "pages" in mimalloc terminology). The +use of QSBR creates a delay between all memory blocks for a page being freed +and the memory page being released, either for new allocations or back to the +OS. + +The mimalloc allocator also defers returning freed memory back to the OS. You +can reduce that delay by setting the environment variable +:envvar:`!MIMALLOC_PURGE_DELAY` to ``0``. Note that this will likely reduce +the performance of the allocator. + + +Free-threaded reference counting can cause objects to live longer +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In the default build, when an object's reference count reaches zero, it is +normally deallocated. The free-threaded build uses "biased reference +counting", with a fast-path for objects "owned" by the current thread and a +slow path for other objects. See :pep:`703` for additional details. Any time +an object's reference count ends up in a "queued" state, deallocation can be +deferred. The queued state is cleared from the "eval breaker" section of the +bytecode evaluator. + +The free-threaded build also allows a different mode of reference counting, +known as "deferred reference counting". This mode is enabled by setting a flag +on a per-object basis. Deferred reference counting is enabled for the +following types: + +* module objects +* module top-level functions +* class methods defined in the class scope +* descriptor objects +* thread-local objects, created by :class:`threading.local` + +When deferred reference counting is enabled, references from Python function +stacks are not added to the reference count. This scheme reduces the overhead +of reference counting, especially for objects used from multiple threads. +Because the stack references are not counted, objects with deferred reference +counting are not immediately freed when their internal reference count goes to +zero. Instead, they are examined by the next GC run and, if no stack +references to them are found, they are freed. This means these objects are +freed by the GC and not when their reference count goes to zero, as is typical. + + +Per-thread reference counting can delay freeing objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To avoid contention on the reference count fields of frequently shared +objects, the free-threaded build also uses "per-thread reference counting" +for a few selected object types. Rather than updating a single shared +reference count, each thread maintains its own local reference count array, +indexed by a unique id assigned to the object. The true reference count is +only computed by summing the per-thread counts when the object's local +count drops to zero. Per-thread reference counting is currently used for: + +* heap type objects (classes created in Python) +* code objects +* the ``__dict__`` of module objects + +Because the per-thread counts must be merged back to the object before it +can be deallocated, objects using per-thread reference counting are +typically freed later than they would be in the default build. In +particular, such an object is usually not freed until the thread that +referenced it reaches a safe point (for example, in the "eval breaker" +section of the bytecode evaluator) or exits. Running :func:`gc.collect` +will merge the per-thread counts and allow these objects to be freed. diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 053558e389030a8..a61fdaee27f6b18 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -4,9 +4,6 @@ Functional Programming HOWTO ******************************** -:Author: A. M. Kuchling -:Release: 0.32 - In this document, we'll take a tour of Python's features suitable for implementing programs in a functional style. After an introduction to the concepts of functional programming, we'll look at language features such as @@ -1042,7 +1039,7 @@ first calculation. :: >>> functools.reduce(operator.concat, []) Traceback (most recent call last): ... - TypeError: reduce() of empty sequence with no initial value + TypeError: reduce() of empty iterable with no initial value >>> functools.reduce(operator.mul, [1, 2, 3], 1) 6 >>> functools.reduce(operator.mul, [], 1) @@ -1185,7 +1182,8 @@ about whether this lambda-free style is better. Revision History and Acknowledgements ===================================== -The author would like to thank the following people for offering suggestions, +This HOWTO was originally written by A. M. Kuchling. The author would like to +thank the following people for offering suggestions, corrections and assistance with various drafts of this article: Ian Bicking, Nick Coghlan, Nick Efford, Raymond Hettinger, Jim Jewett, Mike Krell, Leandro Lameiro, Jussi Salmela, Collin Winter, Blake Winton. @@ -1239,9 +1237,9 @@ Text Processing". Mertz also wrote a 3-part series of articles on functional programming for IBM's DeveloperWorks site; see -`part 1 `__, -`part 2 `__, and -`part 3 `__, +`part 1 `__, +`part 2 `__, and +`part 3 `__. Python documentation diff --git a/Doc/howto/gdb_helpers.rst b/Doc/howto/gdb_helpers.rst index 98ce813ca4ab024..33d1fbf8cd9e9ea 100644 --- a/Doc/howto/gdb_helpers.rst +++ b/Doc/howto/gdb_helpers.rst @@ -136,7 +136,7 @@ enabled:: at Objects/unicodeobject.c:551 #7 0x0000000000440d94 in PyUnicodeUCS2_FromString (u=0x5c2b8d "__lltrace__") at Objects/unicodeobject.c:569 #8 0x0000000000584abd in PyDict_GetItemString (v= - {'Yuck': , '__builtins__': , '__file__': 'Lib/test/crashers/nasty_eq_vs_dict.py', '__package__': None, 'y': , 'dict': {0: 0, 1: 1, 2: 2, 3: 3}, '__cached__': None, '__name__': '__main__', 'z': , '__doc__': None}, key= + {'Yuck': , '__builtins__': , '__file__': 'Lib/test/crashers/nasty_eq_vs_dict.py', '__package__': None, 'y': , 'dict': {0: 0, 1: 1, 2: 2, 3: 3}, '__name__': '__main__', 'z': , '__doc__': None}, key= 0x5c2b8d "__lltrace__") at Objects/dictobject.c:2171 Notice how the dictionary argument to ``PyDict_GetItemString`` is displayed diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index f350141004c2db4..57e2d6e0752447f 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -1,3 +1,5 @@ +.. _how-tos: + *************** Python HOWTOs *************** @@ -11,6 +13,7 @@ Python Library Reference. :maxdepth: 1 :hidden: + a-conceptual-overview-of-asyncio.rst cporting.rst curses.rst descriptor.rst @@ -34,10 +37,12 @@ Python Library Reference. mro.rst free-threading-python.rst free-threading-extensions.rst + abi3t-migration.rst remote_debugging.rst General: +* :ref:`a-conceptual-overview-of-asyncio` * :ref:`annotations-howto` * :ref:`argparse-tutorial` * :ref:`descriptorhowto` @@ -57,6 +62,7 @@ Advanced development: * :ref:`freethreading-python-howto` * :ref:`freethreading-extensions-howto` * :ref:`isolating-extensions-howto` +* :ref:`abi3t-migration-howto` * :ref:`python_2.3_mro` * :ref:`socket-howto` * :ref:`timerfd-howto` diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index 6e03ef20a21fa3f..8f0b0c41ea48617 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -6,9 +6,6 @@ Instrumenting CPython with DTrace and SystemTap =============================================== -:author: David Malcolm -:author: Łukasz Langa - DTrace and SystemTap are monitoring tools, each providing a way to inspect what the processes on a computer system are doing. They both use domain-specific languages allowing a user to write scripts which: @@ -269,6 +266,8 @@ should instead read: (assuming a :ref:`debug build ` of CPython 3.6) +.. _static-markers: + Available static markers ------------------------ @@ -339,6 +338,84 @@ Available static markers .. versionadded:: 3.8 +C Entry Points +^^^^^^^^^^^^^^ + +To simplify triggering of DTrace markers, Python's C API comes with a number +of helper functions that mirror each static marker. On builds of Python without +DTrace enabled, these do nothing. + +In general, it is not necessary to call these yourself, as Python will do +it for you. + +.. list-table:: + :widths: 50 25 25 + :header-rows: 1 + + * * C API Function + * Static Marker + * Notes + * * .. c:function:: void PyDTrace_LINE(const char *arg0, const char *arg1, int arg2) + * :c:func:`!line` + * + * * .. c:function:: void PyDTrace_FUNCTION_ENTRY(const char *arg0, const char *arg1, int arg2) + * :c:func:`!function__entry` + * + * * .. c:function:: void PyDTrace_FUNCTION_RETURN(const char *arg0, const char *arg1, int arg2) + * :c:func:`!function__return` + * + * * .. c:function:: void PyDTrace_GC_START(int arg0) + * :c:func:`!gc__start` + * + * * .. c:function:: void PyDTrace_GC_DONE(Py_ssize_t arg0) + * :c:func:`!gc__done` + * + * * .. c:function:: void PyDTrace_INSTANCE_NEW_START(int arg0) + * :c:func:`!instance__new__start` + * Not used by Python + * * .. c:function:: void PyDTrace_INSTANCE_NEW_DONE(int arg0) + * :c:func:`!instance__new__done` + * Not used by Python + * * .. c:function:: void PyDTrace_INSTANCE_DELETE_START(int arg0) + * :c:func:`!instance__delete__start` + * Not used by Python + * * .. c:function:: void PyDTrace_INSTANCE_DELETE_DONE(int arg0) + * :c:func:`!instance__delete__done` + * Not used by Python + * * .. c:function:: void PyDTrace_IMPORT_FIND_LOAD_START(const char *arg0) + * :c:func:`!import__find__load__start` + * + * * .. c:function:: void PyDTrace_IMPORT_FIND_LOAD_DONE(const char *arg0, int arg1) + * :c:func:`!import__find__load__done` + * + * * .. c:function:: void PyDTrace_AUDIT(const char *arg0, void *arg1) + * :c:func:`!audit` + * + + +C Probing Checks +^^^^^^^^^^^^^^^^ + +.. c:function:: int PyDTrace_LINE_ENABLED(void) +.. c:function:: int PyDTrace_FUNCTION_ENTRY_ENABLED(void) +.. c:function:: int PyDTrace_FUNCTION_RETURN_ENABLED(void) +.. c:function:: int PyDTrace_GC_START_ENABLED(void) +.. c:function:: int PyDTrace_GC_DONE_ENABLED(void) +.. c:function:: int PyDTrace_INSTANCE_NEW_START_ENABLED(void) +.. c:function:: int PyDTrace_INSTANCE_NEW_DONE_ENABLED(void) +.. c:function:: int PyDTrace_INSTANCE_DELETE_START_ENABLED(void) +.. c:function:: int PyDTrace_INSTANCE_DELETE_DONE_ENABLED(void) +.. c:function:: int PyDTrace_IMPORT_FIND_LOAD_START_ENABLED(void) +.. c:function:: int PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED(void) +.. c:function:: int PyDTrace_AUDIT_ENABLED(void) + + All calls to ``PyDTrace`` functions must be guarded by a call to one + of these functions. This allows Python to minimize performance impact + when probing is disabled. + + On builds without DTrace enabled, these functions do nothing and return + ``0``. + SystemTap Tapsets ----------------- diff --git a/Doc/howto/ipaddress.rst b/Doc/howto/ipaddress.rst index e852db98156facc..646c4c4d9e7ab14 100644 --- a/Doc/howto/ipaddress.rst +++ b/Doc/howto/ipaddress.rst @@ -8,9 +8,6 @@ An introduction to the ipaddress module *************************************** -:author: Peter Moody -:author: Nick Coghlan - .. topic:: Overview This document aims to provide a gentle introduction to the diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 7da6dc8a39795e8..6092c75f48fdef0 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -353,7 +353,7 @@ garbage collection protocol. That is, heap types should: - Have the :c:macro:`Py_TPFLAGS_HAVE_GC` flag. -- Define a traverse function using ``Py_tp_traverse``, which +- Define a traverse function using :c:data:`Py_tp_traverse`, which visits the type (e.g. using ``Py_VISIT(Py_TYPE(self))``). Please refer to the documentation of diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 52537a91df542c1..87025814aafb9ab 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -4,8 +4,6 @@ Logging Cookbook ================ -:Author: Vinay Sajip - This page contains a number of recipes related to logging, which have been found useful in the past. For links to tutorial and reference information, please see :ref:`cookbook-ref-links`. @@ -229,7 +227,7 @@ messages should not. Here's how you can achieve this:: # tell the handler to use this format console.setFormatter(formatter) # add the handler to the root logger - logging.getLogger('').addHandler(console) + logging.getLogger().addHandler(console) # Now, we can log to the root logger, or any other logger. First the root... logging.info('Jackdaws love my big sphinx of quartz.') @@ -650,7 +648,7 @@ the receiving end. A simple way of doing this is attaching a import logging, logging.handlers - rootLogger = logging.getLogger('') + rootLogger = logging.getLogger() rootLogger.setLevel(logging.DEBUG) socketHandler = logging.handlers.SocketHandler('localhost', logging.handlers.DEFAULT_TCP_LOGGING_PORT) @@ -1549,10 +1547,10 @@ to this (remembering to first import :mod:`concurrent.futures`):: for i in range(10): executor.submit(worker_process, queue, worker_configurer) -Deploying Web applications using Gunicorn and uWSGI +Deploying web applications using Gunicorn and uWSGI ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -When deploying Web applications using `Gunicorn `_ or `uWSGI +When deploying web applications using `Gunicorn `_ or `uWSGI `_ (or similar), multiple worker processes are created to handle client requests. In such environments, avoid creating file-based handlers directly in your web application. Instead, use a @@ -1564,9 +1562,6 @@ process. This can be set up using a process management tool such as Supervisor - Using file rotation ------------------- -.. sectionauthor:: Doug Hellmann, Vinay Sajip (changes) -.. (see ) - Sometimes you want to let a log file grow to a certain size, then open a new file and log to that. You may want to keep a certain number of these files, and when that many files have been created, rotate the files so that the number of @@ -3619,7 +3614,6 @@ detailed information. .. code-block:: python3 - import datetime import logging import random import sys @@ -3854,7 +3848,7 @@ Logging to syslog with RFC5424 support Although :rfc:`5424` dates from 2009, most syslog servers are configured by default to use the older :rfc:`3164`, which hails from 2001. When ``logging`` was added to Python in 2003, it supported the earlier (and only existing) protocol at the time. Since -RFC5424 came out, as there has not been widespread deployment of it in syslog +RFC 5424 came out, as there has not been widespread deployment of it in syslog servers, the :class:`~logging.handlers.SysLogHandler` functionality has not been updated. @@ -3862,7 +3856,7 @@ RFC 5424 contains some useful features such as support for structured data, and need to be able to log to a syslog server with support for it, you can do so with a subclassed handler which looks something like this:: - import datetime + import datetime as dt import logging.handlers import re import socket @@ -3880,8 +3874,8 @@ subclassed handler which looks something like this:: def format(self, record): version = 1 - asctime = datetime.datetime.fromtimestamp(record.created).isoformat() - m = self.tz_offset.match(time.strftime('%z')) + asctime = dt.datetime.fromtimestamp(record.created).isoformat() + m = self.tz_offset.prefixmatch(time.strftime('%z')) has_offset = False if m and time.timezone: hrs, mins = m.groups() diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index b7225ff1c2cbfc8..c8ce0df9e937f8c 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -4,8 +4,6 @@ Logging HOWTO ============= -:Author: Vinay Sajip - .. _logging-basic-tutorial: .. currentmodule:: logging @@ -28,7 +26,7 @@ When to use logging ^^^^^^^^^^^^^^^^^^^ You can access logging functionality by creating a logger via ``logger = -getLogger(__name__)``, and then calling the logger's :meth:`~Logger.debug`, +logging.getLogger(__name__)``, and then calling the logger's :meth:`~Logger.debug`, :meth:`~Logger.info`, :meth:`~Logger.warning`, :meth:`~Logger.error` and :meth:`~Logger.critical` methods. To determine when to use logging, and to see which logger methods to use when, see the table below. It states, for each of a diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index fc4772bbccab57a..5565f99b244f11b 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -6,8 +6,6 @@ Python support for the ``perf map`` compatible profilers ======================================================== -:author: Pablo Galindo - `The Linux perf profiler `_ and `samply `_ are powerful tools that allow you to profile and obtain information about the performance of your application. @@ -217,8 +215,9 @@ Example, using the :mod:`sys` APIs in file :file:`example.py`: How to obtain the best results ------------------------------ -For best results, Python should be compiled with -``CFLAGS="-fno-omit-frame-pointer -mno-omit-leaf-frame-pointer"`` as this allows +For best results, keep frame pointers enabled. On supported GCC-compatible +toolchains, CPython builds itself with ``-fno-omit-frame-pointer`` and similar +flags (see :option:`--without-frame-pointers` for details). These flags allow profilers to unwind using only the frame pointer and not on DWARF debug information. This is because as the code that is interposed to allow ``perf`` support is dynamically generated it doesn't have any DWARF debugging information diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index 9f73c811cfcbc0d..f19f6006dbac9e7 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -6,8 +6,6 @@ How to port Python 2 Code to Python 3 ************************************* -:author: Brett Cannon - Python 2 reached its official end-of-life at the start of 2020. This means that no new bug reports, fixes, or changes will be made to Python 2 - it's no longer supported: see :pep:`373` and diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index 7486a378dbb06f6..ecdb35136a99f3b 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -1,11 +1,9 @@ .. _regex-howto: **************************** - Regular Expression HOWTO + Regular expression HOWTO **************************** -:Author: A.M. Kuchling - .. TODO: Document lookbehind assertions Better way of displaying a RE, a string, and what it matches @@ -47,7 +45,7 @@ Python code to do the processing; while Python code will be slower than an elaborate regular expression, it will also probably be more understandable. -Simple Patterns +Simple patterns =============== We'll start by learning about the simplest possible regular expressions. Since @@ -59,7 +57,7 @@ expressions (deterministic and non-deterministic finite automata), you can refer to almost any textbook on writing compilers. -Matching Characters +Matching characters ------------------- Most letters and characters will simply match themselves. For example, the @@ -159,7 +157,7 @@ match even a newline. ``.`` is often used where you want to match "any character". -Repeating Things +Repeating things ---------------- Being able to match varying sets of characters is the first thing regular @@ -210,7 +208,7 @@ this RE against the string ``'abcbd'``. | | | ``[bcd]*`` is only matching | | | | ``bc``. | +------+-----------+---------------------------------+ -| 6 | ``abcb`` | Try ``b`` again. This time | +| 7 | ``abcb`` | Try ``b`` again. This time | | | | the character at the | | | | current position is ``'b'``, so | | | | it succeeds. | @@ -255,7 +253,7 @@ is equivalent to ``+``, and ``{0,1}`` is the same as ``?``. It's better to use to read. -Using Regular Expressions +Using regular expressions ========================= Now that we've looked at some simple regular expressions, how do we actually use @@ -264,7 +262,7 @@ expression engine, allowing you to compile REs into objects and then perform matches with them. -Compiling Regular Expressions +Compiling regular expressions ----------------------------- Regular expressions are compiled into pattern objects, which have @@ -295,7 +293,7 @@ disadvantage which is the topic of the next section. .. _the-backslash-plague: -The Backslash Plague +The backslash plague -------------------- As stated earlier, regular expressions use the backslash character (``'\'``) to @@ -335,7 +333,7 @@ expressions will often be written in Python code using this raw string notation. In addition, special escape sequences that are valid in regular expressions, but not valid as Python string literals, now result in a -:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`, +:exc:`SyntaxWarning` and will eventually become a :exc:`SyntaxError`, which means the sequences will be invalid if raw string notation or escaping the backslashes isn't used. @@ -351,7 +349,7 @@ the backslashes isn't used. +-------------------+------------------+ -Performing Matches +Performing matches ------------------ Once you have an object representing a compiled regular expression, what do you @@ -362,20 +360,21 @@ for a complete listing. +------------------+-----------------------------------------------+ | Method/Attribute | Purpose | +==================+===============================================+ -| ``match()`` | Determine if the RE matches at the beginning | -| | of the string. | -+------------------+-----------------------------------------------+ | ``search()`` | Scan through a string, looking for any | | | location where this RE matches. | +------------------+-----------------------------------------------+ +| ``prefixmatch()``| Determine if the RE matches at the beginning | +| | of the string. Previously named :ref:`match() | +| | `. | ++------------------+-----------------------------------------------+ | ``findall()`` | Find all substrings where the RE matches, and | -| | returns them as a list. | +| | return them as a list. | +------------------+-----------------------------------------------+ | ``finditer()`` | Find all substrings where the RE matches, and | -| | returns them as an :term:`iterator`. | +| | return them as an :term:`iterator`. | +------------------+-----------------------------------------------+ -:meth:`~re.Pattern.match` and :meth:`~re.Pattern.search` return ``None`` if no match can be found. If +:meth:`~re.Pattern.search` and :meth:`~re.Pattern.prefixmatch` return ``None`` if no match can be found. If they're successful, a :ref:`match object ` instance is returned, containing information about the match: where it starts and ends, the substring it matched, and more. @@ -393,19 +392,19 @@ Python interpreter, import the :mod:`re` module, and compile a RE:: Now, you can try matching various strings against the RE ``[a-z]+``. An empty string shouldn't match at all, since ``+`` means 'one or more repetitions'. -:meth:`~re.Pattern.match` should return ``None`` in this case, which will cause the +:meth:`~re.Pattern.search` should return ``None`` in this case, which will cause the interpreter to print no output. You can explicitly print the result of -:meth:`!match` to make this clear. :: +:meth:`!search` to make this clear. :: - >>> p.match("") - >>> print(p.match("")) + >>> p.search("") + >>> print(p.search("")) None Now, let's try it on a string that it should match, such as ``tempo``. In this -case, :meth:`~re.Pattern.match` will return a :ref:`match object `, so you +case, :meth:`~re.Pattern.search` will return a :ref:`match object `, so you should store the result in a variable for later use. :: - >>> m = p.match('tempo') + >>> m = p.search('tempo') >>> m @@ -437,27 +436,28 @@ Trying these methods will soon clarify their meaning:: :meth:`~re.Match.group` returns the substring that was matched by the RE. :meth:`~re.Match.start` and :meth:`~re.Match.end` return the starting and ending index of the match. :meth:`~re.Match.span` -returns both start and end indexes in a single tuple. Since the :meth:`~re.Pattern.match` -method only checks if the RE matches at the start of a string, :meth:`!start` -will always be zero. However, the :meth:`~re.Pattern.search` method of patterns -scans through the string, so the match may not start at zero in that -case. :: +returns both start and end indexes in a single tuple. +The :meth:`~re.Pattern.search` method of patterns +scans through the string, so the match may not start at zero. +However, the :meth:`~re.Pattern.prefixmatch` +method only checks if the RE matches at the start of a string, so :meth:`!start` +will always be zero in that case. :: - >>> print(p.match('::: message')) - None >>> m = p.search('::: message'); print(m) >>> m.group() 'message' >>> m.span() (4, 11) + >>> print(p.prefixmatch('::: message')) + None In actual programs, the most common style is to store the :ref:`match object ` in a variable, and then check if it was ``None``. This usually looks like:: p = re.compile( ... ) - m = p.match( 'string goes here' ) + m = p.search( 'string goes here' ) if m: print('Match found: ', m.group()) else: @@ -473,7 +473,7 @@ Two pattern methods return all of the matches for a pattern. The ``r`` prefix, making the literal a raw string literal, is needed in this example because escape sequences in a normal "cooked" string literal that are not recognized by Python, as opposed to regular expressions, now result in a -:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`. See +:exc:`SyntaxWarning` and will eventually become a :exc:`SyntaxError`. See :ref:`the-backslash-plague`. :meth:`~re.Pattern.findall` has to create the entire list before it can be returned as the @@ -491,19 +491,19 @@ result. The :meth:`~re.Pattern.finditer` method returns a sequence of (29, 31) -Module-Level Functions +Module-level functions ---------------------- You don't have to create a pattern object and call its methods; the -:mod:`re` module also provides top-level functions called :func:`~re.match`, -:func:`~re.search`, :func:`~re.findall`, :func:`~re.sub`, and so forth. These functions +:mod:`re` module also provides top-level functions called :func:`~re.search`, +:func:`~re.prefixmatch`, :func:`~re.findall`, :func:`~re.sub`, and so forth. These functions take the same arguments as the corresponding pattern method with the RE string added as the first argument, and still return either ``None`` or a :ref:`match object ` instance. :: - >>> print(re.match(r'From\s+', 'Fromage amk')) + >>> print(re.prefixmatch(r'From\s+', 'Fromage amk')) None - >>> re.match(r'From\s+', 'From amk Thu May 14 19:12:10 1998') #doctest: +ELLIPSIS + >>> re.prefixmatch(r'From\s+', 'From amk Thu May 14 19:12:10 1998') #doctest: +ELLIPSIS Under the hood, these functions simply create a pattern object for you @@ -518,7 +518,7 @@ Outside of loops, there's not much difference thanks to the internal cache. -Compilation Flags +Compilation flags ----------------- .. currentmodule:: re @@ -642,7 +642,7 @@ of each one. whitespace is in a character class or preceded by an unescaped backslash; this lets you organize and indent the RE more clearly. This flag also lets you put comments within a RE that will be ignored by the engine; comments are marked by - a ``'#'`` that's neither in a character class or preceded by an unescaped + a ``'#'`` that's neither in a character class nor preceded by an unescaped backslash. For example, here's a RE that uses :const:`re.VERBOSE`; see how much easier it @@ -669,7 +669,7 @@ of each one. to understand than the version using :const:`re.VERBOSE`. -More Pattern Power +More pattern power ================== So far we've only covered a part of the features of regular expressions. In @@ -679,7 +679,7 @@ retrieve portions of the text that was matched. .. _more-metacharacters: -More Metacharacters +More metacharacters ------------------- There are some metacharacters that we haven't covered yet. Most of them will be @@ -812,7 +812,7 @@ of a group with a quantifier, such as ``*``, ``+``, ``?``, or ``ab``. :: >>> p = re.compile('(ab)*') - >>> print(p.match('ababababab').span()) + >>> print(p.search('ababababab').span()) (0, 10) Groups indicated with ``'('``, ``')'`` also capture the starting and ending @@ -825,7 +825,7 @@ argument. Later we'll see how to express groups that don't capture the span of text that they match. :: >>> p = re.compile('(a)b') - >>> m = p.match('ab') + >>> m = p.search('ab') >>> m.group() 'ab' >>> m.group(0) @@ -836,7 +836,7 @@ to determine the number, just count the opening parenthesis characters, going from left to right. :: >>> p = re.compile('(a(b)c)d') - >>> m = p.match('abcd') + >>> m = p.search('abcd') >>> m.group(0) 'abcd' >>> m.group(1) @@ -875,7 +875,7 @@ Backreferences like this aren't often useful for just searching through a string find out that they're *very* useful when performing string substitutions. -Non-capturing and Named Groups +Non-capturing and named groups ------------------------------ Elaborate REs may use many groups, both to capture substrings of interest, and @@ -912,10 +912,10 @@ but aren't interested in retrieving the group's contents. You can make this fact explicit by using a non-capturing group: ``(?:...)``, where you can replace the ``...`` with any other regular expression. :: - >>> m = re.match("([abc])+", "abc") + >>> m = re.search("([abc])+", "abc") >>> m.groups() ('c',) - >>> m = re.match("(?:[abc])+", "abc") + >>> m = re.search("(?:[abc])+", "abc") >>> m.groups() () @@ -949,7 +949,7 @@ given numbers, so you can retrieve information about a group in two ways:: Additionally, you can retrieve named groups as a dictionary with :meth:`~re.Match.groupdict`:: - >>> m = re.match(r'(?P\w+) (?P\w+)', 'Jane Doe') + >>> m = re.search(r'(?P\w+) (?P\w+)', 'Jane Doe') >>> m.groupdict() {'first': 'Jane', 'last': 'Doe'} @@ -979,7 +979,7 @@ current point. The regular expression for finding doubled words, 'the the' -Lookahead Assertions +Lookahead assertions -------------------- Another zero-width assertion is the lookahead assertion. Lookahead assertions @@ -1061,7 +1061,7 @@ end in either ``bat`` or ``exe``: ``.*[.](?!bat$|exe$)[^.]*$`` -Modifying Strings +Modifying strings ================= Up to this point, we've simply performed searches against a static string. @@ -1083,7 +1083,7 @@ using the following pattern methods: +------------------+-----------------------------------------------+ -Splitting Strings +Splitting strings ----------------- The :meth:`~re.Pattern.split` method of a pattern splits a string apart @@ -1137,7 +1137,7 @@ argument, but is otherwise the same. :: ['Words', 'words, words.'] -Search and Replace +Search and replace ------------------ Another common task is to find all the matches for a pattern, and replace them @@ -1236,7 +1236,7 @@ pattern object as the first parameter, or use embedded modifiers in the pattern string, e.g. ``sub("(?i)b+", "x", "bbbb BBBB")`` returns ``'x x'``. -Common Problems +Common problems =============== Regular expressions are a powerful tool for some applications, but in some ways @@ -1244,7 +1244,7 @@ their behaviour isn't intuitive and at times they don't behave the way you may expect them to. This section will point out some of the most common pitfalls. -Use String Methods +Use string methods ------------------ Sometimes using the :mod:`re` module is a mistake. If you're matching a fixed @@ -1274,21 +1274,26 @@ In short, before turning to the :mod:`re` module, consider whether your problem can be solved with a faster and simpler string method. -match() versus search() ------------------------ +.. _match-versus-search: + +prefixmatch() (aka match) versus search() +----------------------------------------- -The :func:`~re.match` function only checks if the RE matches at the beginning of the -string while :func:`~re.search` will scan forward through the string for a match. -It's important to keep this distinction in mind. Remember, :func:`!match` will -only report a successful match which will start at 0; if the match wouldn't -start at zero, :func:`!match` will *not* report it. :: +:func:`~re.prefixmatch` was added in Python 3.15 as the :ref:`preferred name +` for :func:`~re.match`. Before this, it was only known +as :func:`!match` and the distinction with :func:`~re.search` was often +misunderstood. - >>> print(re.match('super', 'superstition').span()) +:func:`!prefixmatch` aka :func:`!match` only checks if the RE matches at the +beginning of the string while :func:`!search` scans forward through the +string for a match. :: + + >>> print(re.prefixmatch('super', 'superstition').span()) (0, 5) - >>> print(re.match('super', 'insuperable')) + >>> print(re.prefixmatch('super', 'insuperable')) None -On the other hand, :func:`~re.search` will scan forward through the string, +On the other hand, :func:`~re.search` scans forward through the string, reporting the first match it finds. :: >>> print(re.search('super', 'superstition').span()) @@ -1296,21 +1301,11 @@ reporting the first match it finds. :: >>> print(re.search('super', 'insuperable').span()) (2, 7) -Sometimes you'll be tempted to keep using :func:`re.match`, and just add ``.*`` -to the front of your RE. Resist this temptation and use :func:`re.search` -instead. The regular expression compiler does some analysis of REs in order to -speed up the process of looking for a match. One such analysis figures out what -the first character of a match must be; for example, a pattern starting with -``Crow`` must match starting with a ``'C'``. The analysis lets the engine -quickly scan through the string looking for the starting character, only trying -the full match if a ``'C'`` is found. - -Adding ``.*`` defeats this optimization, requiring scanning to the end of the -string and then backtracking to find a match for the rest of the RE. Use -:func:`re.search` instead. +This distinction is important to remember when using the old :func:`~re.match` +name in code requiring compatibility with older Python versions. -Greedy versus Non-Greedy +Greedy versus non-greedy ------------------------ When repeating a regular expression, as in ``a*``, the resulting action is to @@ -1322,9 +1317,9 @@ doesn't work because of the greedy nature of ``.*``. :: >>> s = 'Title' >>> len(s) 32 - >>> print(re.match('<.*>', s).span()) + >>> print(re.prefixmatch('<.*>', s).span()) (0, 32) - >>> print(re.match('<.*>', s).group()) + >>> print(re.prefixmatch('<.*>', s).group()) Title The RE matches the ``'<'`` in ``''``, and the ``.*`` consumes the rest of @@ -1340,7 +1335,7 @@ example, the ``'>'`` is tried immediately after the first ``'<'`` matches, and when it fails, the engine advances a character at a time, retrying the ``'>'`` at every step. This produces just the right result:: - >>> print(re.match('<.*?>', s).group()) + >>> print(re.prefixmatch('<.*?>', s).group()) (Note that parsing HTML or XML with regular expressions is painful. @@ -1388,9 +1383,9 @@ Feedback ======== Regular expressions are a complicated topic. Did this document help you -understand them? Were there parts that were unclear, or Problems you +understand them? Were there parts that were unclear, or problems you encountered that weren't covered here? If so, please send suggestions for -improvements to the author. +improvements to the :ref:`issue tracker `. The most complete book on regular expressions is almost certainly Jeffrey Friedl's Mastering Regular Expressions, published by O'Reilly. Unfortunately, diff --git a/Doc/howto/remote_debugging.rst b/Doc/howto/remote_debugging.rst index 3adb6ad03e54452..1d5cf24d0628432 100644 --- a/Doc/howto/remote_debugging.rst +++ b/Doc/howto/remote_debugging.rst @@ -3,6 +3,88 @@ Remote debugging attachment protocol ==================================== +This protocol enables external tools to attach to a running CPython process and +execute Python code remotely. + +Most platforms require elevated privileges to attach to another Python process. + +Disabling remote debugging +-------------------------- + +To disable remote debugging support, use any of the following: + +* Set the :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` environment variable to ``1`` before + starting the interpreter. +* Use the :option:`-X disable_remote_debug` command-line option. +* Compile Python with the :option:`--without-remote-debug` build flag. + +.. _permission-requirements: + +Permission requirements +======================= + +Attaching to a running Python process for remote debugging requires elevated +privileges on most platforms. The specific requirements and troubleshooting +steps depend on your operating system: + +.. rubric:: Linux + +The tracer process must have the ``CAP_SYS_PTRACE`` capability or equivalent +privileges. You can only trace processes you own and can signal. Tracing may +fail if the process is already being traced, or if it is running with +set-user-ID or set-group-ID. Security modules like Yama may further restrict +tracing. + +To temporarily relax ptrace restrictions (until reboot), run: + + ``echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope`` + +.. note:: + + Disabling ``ptrace_scope`` reduces system hardening and should only be done + in trusted environments. + +If running inside a container, use ``--cap-add=SYS_PTRACE`` or +``--privileged``, and run as root if needed. + +Try re-running the command with elevated privileges: + + ``sudo -E !!`` + + +.. rubric:: macOS + +To attach to another process, you typically need to run your debugging tool +with elevated privileges. This can be done by using ``sudo`` or running as +root. + +Even when attaching to processes you own, macOS may block debugging unless +the debugger is run with root privileges due to system security restrictions. + + +.. rubric:: Windows + +To attach to another process, you usually need to run your debugging tool +with administrative privileges. Start the command prompt or terminal as +Administrator. + +Some processes may still be inaccessible even with Administrator rights, +unless you have the ``SeDebugPrivilege`` privilege enabled. + +To resolve file or folder access issues, adjust the security permissions: + + 1. Right-click the file or folder and select **Properties**. + 2. Go to the **Security** tab to view users and groups with access. + 3. Click **Edit** to modify permissions. + 4. Select your user account. + 5. In **Permissions**, check **Read** or **Full control** as needed. + 6. Click **Apply**, then **OK** to confirm. + + +.. note:: + + Ensure you've satisfied all :ref:`permission-requirements` before proceeding. + This section describes the low-level protocol that enables external tools to inject and execute a Python script within a running CPython process. @@ -374,13 +456,13 @@ To locate a thread: reliable thread to target. 3. Optionally, use the offset ``interpreter_state.threads_head`` to iterate -through the linked list of all thread states. Each ``PyThreadState`` structure -contains a ``native_thread_id`` field, which may be compared to a target thread -ID to find a specific thread. + through the linked list of all thread states. Each ``PyThreadState`` + structure contains a ``native_thread_id`` field, which may be compared to + a target thread ID to find a specific thread. -1. Once a valid ``PyThreadState`` has been found, its address can be used in -later steps of the protocol, such as writing debugger control fields and -scheduling execution. +4. Once a valid ``PyThreadState`` has been found, its address can be used in + later steps of the protocol, such as writing debugger control fields and + scheduling execution. The following is an example implementation that locates the main thread state:: @@ -454,15 +536,15 @@ its fields are defined by the ``_Py_DebugOffsets`` structure and include the following: - ``debugger_script_path``: A fixed-size buffer that holds the full path to a - Python source file (``.py``). This file must be accessible and readable by - the target process when execution is triggered. + Python source file (``.py``). This file must be accessible and readable by + the target process when execution is triggered. - ``debugger_pending_call``: An integer flag. Setting this to ``1`` tells the - interpreter that a script is ready to be executed. + interpreter that a script is ready to be executed. - ``eval_breaker``: A field checked by the interpreter during execution. - Setting bit 5 (``_PY_EVAL_PLEASE_STOP_BIT``, value ``1U << 5``) in this - field causes the interpreter to pause and check for debugger activity. + Setting bit 5 (``_PY_EVAL_PLEASE_STOP_BIT``, value ``1U << 5``) in this + field causes the interpreter to pause and check for debugger activity. To complete the injection, the debugger must perform the following steps: @@ -543,3 +625,57 @@ To inject and execute a Python script in a remote process: 7. Resume the process (if suspended). The script will execute at the next safe evaluation point. +.. _remote-debugging-threat-model: + +Security and threat model +========================= + +The remote debugging protocol relies on the same operating system primitives +used by native debuggers such as GDB and LLDB. Attaching to a process +requires the **same privileges** that those debuggers require, for example +``ptrace`` / Yama LSM on Linux, ``task_for_pid`` on macOS, and +``SeDebugPrivilege`` on Windows. Python does not introduce any new privilege +escalation path; if an attacker already possesses the permissions needed to +attach to a process, they could equally use GDB to read memory or inject +code. + +The following principles define what is, and is not, considered a security +vulnerability in this feature: + +Attaching requires OS-level privileges + On every supported platform the operating system gates cross-process + memory access behind privilege checks (``CAP_SYS_PTRACE``, root, or + administrator rights). A report that demonstrates an issue only after + these privileges have already been obtained is **not** a vulnerability in + CPython, since the OS security boundary was already crossed. + +Crashes or memory errors when reading a compromised process are not vulnerabilities + A tool that reads internal interpreter state from a target process must + trust that memory to be well-formed. If the target process has been + corrupted or is controlled by an attacker, the debugger or profiler may + crash, produce garbage output, or behave unpredictably. This is the same + risk accepted by every ``ptrace``-based debugger. Bugs in this category + (buffer overflows, segmentation faults, or undefined behaviour triggered + by reading corrupted state) are **not** treated as security issues, though + fixes that improve robustness are welcome. + +Vulnerabilities in the target process are not in scope + If the Python process being debugged has already been compromised, the + attacker already controls execution in that process. Demonstrating further + impact from that starting point does not constitute a vulnerability in the + remote debugging protocol. + +When to use ``PYTHON_DISABLE_REMOTE_DEBUG`` +------------------------------------------- + +The environment variable :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` (and the +equivalent :option:`-X disable_remote_debug` flag) allows operators to disable +the in-process side of the protocol as a **defence-in-depth** measure. This +may be useful in hardened or sandboxed deployment environments where no +debugging or profiling of the process is expected and reducing attack surface +is a priority, even though the OS-level privilege checks already prevent +unprivileged access. + +Setting this variable does **not** affect other OS-level debugging interfaces +(``ptrace``, ``/proc``, ``task_for_pid``, etc.), which remain available +according to their own permission models. diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst index cbc49d15a0771b9..b17ab3f4391dad0 100644 --- a/Doc/howto/sockets.rst +++ b/Doc/howto/sockets.rst @@ -4,9 +4,6 @@ Socket Programming HOWTO **************************** -:Author: Gordon McMillan - - .. topic:: Abstract Sockets are used nearly everywhere, but are one of the most severely diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst index 70c34cde8a06592..b511aa4e59fe1e7 100644 --- a/Doc/howto/sorting.rst +++ b/Doc/howto/sorting.rst @@ -3,9 +3,6 @@ Sorting Techniques ****************** -:Author: Andrew Dalke and Raymond Hettinger - - Python lists have a built-in :meth:`list.sort` method that modifies the list in-place. There is also a :func:`sorted` built-in function that builds a new sorted list from an iterable. @@ -375,7 +372,7 @@ Odds and Ends :meth:`~object.__lt__` is not implemented (see :func:`object.__lt__` for details on the mechanics). To avoid surprises, :pep:`8` recommends that all six comparison methods be implemented. - The :func:`~functools.total_ordering` decorator is provided to make that + The :deco:`~functools.total_ordering` decorator is provided to make that task easier. * Key functions need not depend directly on the objects being sorted. A key diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index 254fe7293553535..cbd53d5b619f5a3 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -4,8 +4,6 @@ Unicode HOWTO ***************** -:Release: 1.12 - This HOWTO discusses Python's support for the Unicode specification for representing textual data, and explains various problems that people commonly encounter when trying to work with Unicode. @@ -352,6 +350,8 @@ If you don't include such a comment, the default encoding used will be UTF-8 as already mentioned. See also :pep:`263` for more information. +.. _unicode-properties: + Unicode Properties ------------------ diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index d79d1abe8d05770..e4f218f088ba89d 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -4,9 +4,6 @@ HOWTO Fetch Internet Resources Using The urllib Package *********************************************************** -:Author: `Michael Foord `_ - - Introduction ============ @@ -15,7 +12,7 @@ Introduction You may also find useful the following article on fetching web resources with Python: - * `Basic Authentication `_ + * `Basic Authentication `__ A tutorial on *Basic Authentication*, with examples in Python. diff --git a/Doc/improve-page-nojs.rst b/Doc/improve-page-nojs.rst new file mode 100644 index 000000000000000..91b3a88b95d38b2 --- /dev/null +++ b/Doc/improve-page-nojs.rst @@ -0,0 +1,29 @@ +:orphan: + +**************************** +Improve a documentation page +**************************** + +.. This is the no-javascript version of this page. The one most people + will see (with JavaScript enabled) is improve-page.rst. If you edit + this page, please also edit that one, and vice versa. + +.. only:: html and not epub + +We are always interested to hear ideas about improvements to the documentation. + +.. only:: translation + + If the bug or suggested improvement concerns the translation of this + documentation, open an issue or edit the page in + `translation's repository `_ instead. + +You have a few ways to ask questions or suggest changes: + +- You can start a discussion about the page on the Python discussion forum. + This link will start a topic in the Documentation category: + `New Documentation topic `_. + +- You can open an issue on the Python GitHub issue tracker. This link will + create a new issue with the "docs" label: + `New docs issue `_. diff --git a/Doc/improve-page.rst b/Doc/improve-page.rst new file mode 100644 index 000000000000000..dc89fcb22fbb59c --- /dev/null +++ b/Doc/improve-page.rst @@ -0,0 +1,65 @@ +:orphan: + +**************************** +Improve a documentation page +**************************** + +.. This is the JavaScript-enabled version of this page. Another version + (for those with JavaScript disabled) is improve-page-nojs.rst. If you + edit this page, please also edit that one, and vice versa. + +.. only:: html and not epub + + .. raw:: html + + + +We are always interested to hear ideas about improvements to the documentation. + +You were reading "PAGETITLE" at ``_. The source for that page is on +`GitHub `_. + +.. only:: translation + + If the bug or suggested improvement concerns the translation of this + documentation, open an issue or edit the page in + `translation's repository `_ instead. + +You have a few ways to ask questions or suggest changes: + +- You can start a discussion about the page on the Python discussion forum. + This link will start a pre-populated topic: + `Question about page "PAGETITLE" `_. + +- You can open an issue on the Python GitHub issue tracker. This link will + create a new pre-populated issue: + `Docs: problem with page "PAGETITLE" `_. + +- You can `edit the page on GitHub `_ + to open a pull request and begin the contribution process. diff --git a/Doc/includes/capi-extension/spammodule-01.c b/Doc/includes/capi-extension/spammodule-01.c new file mode 100644 index 000000000000000..8ddb416e7d6cd08 --- /dev/null +++ b/Doc/includes/capi-extension/spammodule-01.c @@ -0,0 +1,58 @@ +/* This file needs to be kept in sync with the tutorial + * at Doc/extending/first-extension-module.rst + */ + +/// Includes + +#include +#include // for system() + +/// Implementation of spam.system + +static PyObject * +spam_system(PyObject *self, PyObject *arg) +{ + const char *command = PyUnicode_AsUTF8AndSize(arg, NULL); + if (command == NULL) { + return NULL; + } + int status = system(command); + PyObject *result = PyLong_FromLong(status); + return result; +} + +/// Module method table + +static PyMethodDef spam_methods[] = { + { + .ml_name="system", + .ml_meth=spam_system, + .ml_flags=METH_O, + .ml_doc="Execute a shell command.", + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +/// Module slot table + +PyABIInfo_VAR(abi_info); + +static PySlot spam_slots[] = { + PySlot_STATIC_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_name, "spam"), + PySlot_STATIC_DATA(Py_mod_doc, "A wonderful module with an example function"), + PySlot_STATIC_DATA(Py_mod_methods, spam_methods), + PySlot_END +}; + +/// Export hook prototype + +PyMODEXPORT_FUNC PyModExport_spam(void); + +/// Module export hook + +PyMODEXPORT_FUNC +PyModExport_spam(void) +{ + return spam_slots; +} diff --git a/Doc/includes/diff.py b/Doc/includes/diff.py index 001619f5f83fc08..bc4bd58ff3e3f11 100644 --- a/Doc/includes/diff.py +++ b/Doc/includes/diff.py @@ -1,4 +1,4 @@ -""" Command line interface to difflib.py providing diffs in four formats: +""" Command-line interface to difflib.py providing diffs in four formats: * ndiff: lists every line and highlights interline changes. * context: highlights clusters of changes in a before/after format. @@ -8,11 +8,11 @@ """ import sys, os, difflib, argparse -from datetime import datetime, timezone +import datetime as dt def file_mtime(path): - t = datetime.fromtimestamp(os.stat(path).st_mtime, - timezone.utc) + t = dt.datetime.fromtimestamp(os.stat(path).st_mtime, + dt.timezone.utc) return t.astimezone().isoformat() def main(): diff --git a/Doc/includes/optional-module.rst b/Doc/includes/optional-module.rst new file mode 100644 index 000000000000000..262e73f2eaa09fb --- /dev/null +++ b/Doc/includes/optional-module.rst @@ -0,0 +1,9 @@ +This is an :term:`optional module`. +If it is missing from your copy of CPython, +look for documentation from your distributor (that is, +whoever provided Python to you). +If you are the distributor, see :ref:`optional-module-requirements`. + +.. Similar notes appear in the docs of the modules: + - zipfile + - tarfile diff --git a/Doc/includes/tzinfo_examples.py b/Doc/includes/tzinfo_examples.py index 1fa6e615e46a763..762b1b62fc871d4 100644 --- a/Doc/includes/tzinfo_examples.py +++ b/Doc/includes/tzinfo_examples.py @@ -1,68 +1,70 @@ -from datetime import tzinfo, timedelta, datetime - -ZERO = timedelta(0) -HOUR = timedelta(hours=1) -SECOND = timedelta(seconds=1) +import datetime as dt # A class capturing the platform's idea of local time. # (May result in wrong values on historical times in # timezones where UTC offset and/or the DST rules had # changed in the past.) -import time as _time +import time + +ZERO = dt.timedelta(0) +HOUR = dt.timedelta(hours=1) +SECOND = dt.timedelta(seconds=1) -STDOFFSET = timedelta(seconds = -_time.timezone) -if _time.daylight: - DSTOFFSET = timedelta(seconds = -_time.altzone) +STDOFFSET = dt.timedelta(seconds=-time.timezone) +if time.daylight: + DSTOFFSET = dt.timedelta(seconds=-time.altzone) else: DSTOFFSET = STDOFFSET DSTDIFF = DSTOFFSET - STDOFFSET -class LocalTimezone(tzinfo): - def fromutc(self, dt): - assert dt.tzinfo is self - stamp = (dt - datetime(1970, 1, 1, tzinfo=self)) // SECOND - args = _time.localtime(stamp)[:6] +class LocalTimezone(dt.tzinfo): + + def fromutc(self, when): + assert when.tzinfo is self + stamp = (when - dt.datetime(1970, 1, 1, tzinfo=self)) // SECOND + args = time.localtime(stamp)[:6] dst_diff = DSTDIFF // SECOND # Detect fold - fold = (args == _time.localtime(stamp - dst_diff)) - return datetime(*args, microsecond=dt.microsecond, - tzinfo=self, fold=fold) + fold = (args == time.localtime(stamp - dst_diff)) + return dt.datetime(*args, microsecond=when.microsecond, + tzinfo=self, fold=fold) - def utcoffset(self, dt): - if self._isdst(dt): + def utcoffset(self, when): + if self._isdst(when): return DSTOFFSET else: return STDOFFSET - def dst(self, dt): - if self._isdst(dt): + def dst(self, when): + if self._isdst(when): return DSTDIFF else: return ZERO - def tzname(self, dt): - return _time.tzname[self._isdst(dt)] + def tzname(self, when): + return time.tzname[self._isdst(when)] - def _isdst(self, dt): - tt = (dt.year, dt.month, dt.day, - dt.hour, dt.minute, dt.second, - dt.weekday(), 0, 0) - stamp = _time.mktime(tt) - tt = _time.localtime(stamp) + def _isdst(self, when): + tt = (when.year, when.month, when.day, + when.hour, when.minute, when.second, + when.weekday(), 0, 0) + stamp = time.mktime(tt) + tt = time.localtime(stamp) return tt.tm_isdst > 0 + Local = LocalTimezone() # A complete implementation of current DST rules for major US time zones. -def first_sunday_on_or_after(dt): - days_to_go = 6 - dt.weekday() +def first_sunday_on_or_after(when): + days_to_go = 6 - when.weekday() if days_to_go: - dt += timedelta(days_to_go) - return dt + when += dt.timedelta(days_to_go) + return when # US DST Rules @@ -75,21 +77,22 @@ def first_sunday_on_or_after(dt): # # In the US, since 2007, DST starts at 2am (standard time) on the second # Sunday in March, which is the first Sunday on or after Mar 8. -DSTSTART_2007 = datetime(1, 3, 8, 2) +DSTSTART_2007 = dt.datetime(1, 3, 8, 2) # and ends at 2am (DST time) on the first Sunday of Nov. -DSTEND_2007 = datetime(1, 11, 1, 2) +DSTEND_2007 = dt.datetime(1, 11, 1, 2) # From 1987 to 2006, DST used to start at 2am (standard time) on the first # Sunday in April and to end at 2am (DST time) on the last # Sunday of October, which is the first Sunday on or after Oct 25. -DSTSTART_1987_2006 = datetime(1, 4, 1, 2) -DSTEND_1987_2006 = datetime(1, 10, 25, 2) +DSTSTART_1987_2006 = dt.datetime(1, 4, 1, 2) +DSTEND_1987_2006 = dt.datetime(1, 10, 25, 2) # From 1967 to 1986, DST used to start at 2am (standard time) on the last # Sunday in April (the one on or after April 24) and to end at 2am (DST time) # on the last Sunday of October, which is the first Sunday # on or after Oct 25. -DSTSTART_1967_1986 = datetime(1, 4, 24, 2) +DSTSTART_1967_1986 = dt.datetime(1, 4, 24, 2) DSTEND_1967_1986 = DSTEND_1987_2006 + def us_dst_range(year): # Find start and end times for US DST. For years before 1967, return # start = end for no DST. @@ -100,17 +103,17 @@ def us_dst_range(year): elif 1966 < year < 1987: dststart, dstend = DSTSTART_1967_1986, DSTEND_1967_1986 else: - return (datetime(year, 1, 1), ) * 2 + return (dt.datetime(year, 1, 1), ) * 2 start = first_sunday_on_or_after(dststart.replace(year=year)) end = first_sunday_on_or_after(dstend.replace(year=year)) return start, end -class USTimeZone(tzinfo): +class USTimeZone(dt.tzinfo): def __init__(self, hours, reprname, stdname, dstname): - self.stdoffset = timedelta(hours=hours) + self.stdoffset = dt.timedelta(hours=hours) self.reprname = reprname self.stdname = stdname self.dstname = dstname @@ -118,45 +121,45 @@ def __init__(self, hours, reprname, stdname, dstname): def __repr__(self): return self.reprname - def tzname(self, dt): - if self.dst(dt): + def tzname(self, when): + if self.dst(when): return self.dstname else: return self.stdname - def utcoffset(self, dt): - return self.stdoffset + self.dst(dt) + def utcoffset(self, when): + return self.stdoffset + self.dst(when) - def dst(self, dt): - if dt is None or dt.tzinfo is None: + def dst(self, when): + if when is None or when.tzinfo is None: # An exception may be sensible here, in one or both cases. # It depends on how you want to treat them. The default # fromutc() implementation (called by the default astimezone() - # implementation) passes a datetime with dt.tzinfo is self. + # implementation) passes a datetime with when.tzinfo is self. return ZERO - assert dt.tzinfo is self - start, end = us_dst_range(dt.year) + assert when.tzinfo is self + start, end = us_dst_range(when.year) # Can't compare naive to aware objects, so strip the timezone from - # dt first. - dt = dt.replace(tzinfo=None) - if start + HOUR <= dt < end - HOUR: + # when first. + when = when.replace(tzinfo=None) + if start + HOUR <= when < end - HOUR: # DST is in effect. return HOUR - if end - HOUR <= dt < end: - # Fold (an ambiguous hour): use dt.fold to disambiguate. - return ZERO if dt.fold else HOUR - if start <= dt < start + HOUR: + if end - HOUR <= when < end: + # Fold (an ambiguous hour): use when.fold to disambiguate. + return ZERO if when.fold else HOUR + if start <= when < start + HOUR: # Gap (a non-existent hour): reverse the fold rule. - return HOUR if dt.fold else ZERO + return HOUR if when.fold else ZERO # DST is off. return ZERO - def fromutc(self, dt): - assert dt.tzinfo is self - start, end = us_dst_range(dt.year) + def fromutc(self, when): + assert when.tzinfo is self + start, end = us_dst_range(when.year) start = start.replace(tzinfo=self) end = end.replace(tzinfo=self) - std_time = dt + self.stdoffset + std_time = when + self.stdoffset dst_time = std_time + HOUR if end <= dst_time < end + HOUR: # Repeated hour diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index 3a485a43a5a7518..c372d9f47418001 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -1,16 +1,14 @@ -.. highlight:: none +.. highlight:: shell .. _installing-index: ************************* -Installing Python Modules +Installing Python modules ************************* -:Email: distutils-sig@python.org - As a popular open source development project, Python has an active supporting community of contributors and users that also make their software -available for other Python developers to use under open source license terms. +available for other Python developers to use under open-source license terms. This allows Python users to share and collaborate effectively, benefiting from the solutions others have already created to common (and sometimes @@ -34,34 +32,24 @@ creating and sharing your own Python projects, refer to the Key terms ========= -* ``pip`` is the preferred installer program. Starting with Python 3.4, it +* :program:`pip` is the preferred installer program. It is included by default with the Python binary installers. * A *virtual environment* is a semi-isolated Python environment that allows packages to be installed for use by a particular application, rather than being installed system wide. -* ``venv`` is the standard tool for creating virtual environments, and has - been part of Python since Python 3.3. Starting with Python 3.4, it - defaults to installing ``pip`` into all created virtual environments. -* ``virtualenv`` is a third party alternative (and predecessor) to - ``venv``. It allows virtual environments to be used on versions of - Python prior to 3.4, which either don't provide ``venv`` at all, or - aren't able to automatically install ``pip`` into created environments. -* The `Python Package Index `__ is a public +* ``venv`` is the standard tool for creating virtual environments. + It defaults to installing :program:`pip` into all created virtual environments. +* ``virtualenv`` is a third-party alternative (and predecessor) to + ``venv``. +* The `Python Package Index (PyPI) `__ is a public repository of open source licensed packages made available for use by other Python users. -* the `Python Packaging Authority +* The `Python Packaging Authority `__ is the group of developers and documentation authors responsible for the maintenance and evolution of the standard packaging tools and the associated metadata and file format standards. They maintain a variety of tools, documentation, and issue trackers on `GitHub `__. -* ``distutils`` is the original build and distribution system first added to - the Python standard library in 1998. While direct use of ``distutils`` is - being phased out, it still laid the foundation for the current packaging - and distribution infrastructure, and it not only remains part of the - standard library, but its name lives on in other ways (such as the name - of the mailing list used to coordinate Python packaging standards - development). .. versionchanged:: 3.5 The use of ``venv`` is now recommended for creating virtual environments. @@ -79,7 +67,7 @@ The standard packaging tools are all designed to be used from the command line. The following command will install the latest version of a module and its -dependencies from the Python Package Index:: +dependencies from PyPI:: python -m pip install SomePackage @@ -106,7 +94,7 @@ explicitly:: python -m pip install --upgrade SomePackage -More information and resources regarding ``pip`` and its capabilities can be +More information and resources regarding :program:`pip` and its capabilities can be found in the `Python Packaging User Guide `__. Creation of virtual environments is done through the :mod:`venv` module. @@ -124,19 +112,6 @@ How do I ...? These are quick answers or links for some common tasks. -... install ``pip`` in versions of Python prior to Python 3.4? --------------------------------------------------------------- - -Python only started bundling ``pip`` with Python 3.4. For earlier versions, -``pip`` needs to be "bootstrapped" as described in the Python Packaging -User Guide. - -.. seealso:: - - `Python Packaging User Guide: Requirements for Installing Packages - `__ - - .. installing-per-user-installation: ... install packages just for the current user? @@ -150,10 +125,10 @@ package just for the current user, rather than for all users of the system. --------------------------------------- A number of scientific Python packages have complex binary dependencies, and -aren't currently easy to install using ``pip`` directly. At this point in -time, it will often be easier for users to install these packages by +aren't currently easy to install using :program:`pip` directly. +It will often be easier for users to install these packages by `other means `__ -rather than attempting to install them with ``pip``. +rather than attempting to install them with :program:`pip`. .. seealso:: @@ -166,22 +141,18 @@ rather than attempting to install them with ``pip``. On Linux, macOS, and other POSIX systems, use the versioned Python commands in combination with the ``-m`` switch to run the appropriate copy of -``pip``:: +:program:`pip`:: - python2 -m pip install SomePackage # default Python 2 - python2.7 -m pip install SomePackage # specifically Python 2.7 - python3 -m pip install SomePackage # default Python 3 - python3.4 -m pip install SomePackage # specifically Python 3.4 + python3 -m pip install SomePackage # default Python 3 + python3.14 -m pip install SomePackage # specifically Python 3.14 -Appropriately versioned ``pip`` commands may also be available. +Appropriately versioned :program:`pip` commands may also be available. -On Windows, use the ``py`` Python launcher in combination with the ``-m`` +On Windows, use the :program:`py` Python launcher in combination with the ``-m`` switch:: - py -2 -m pip install SomePackage # default Python 2 - py -2.7 -m pip install SomePackage # specifically Python 2.7 - py -3 -m pip install SomePackage # default Python 3 - py -3.4 -m pip install SomePackage # specifically Python 3.4 + py -3 -m pip install SomePackage # default Python 3 + py -3.14 -m pip install SomePackage # specifically Python 3.14 .. other questions: @@ -201,39 +172,38 @@ On Linux systems, a Python installation will typically be included as part of the distribution. Installing into this Python installation requires root access to the system, and may interfere with the operation of the system package manager and other components of the system if a component -is unexpectedly upgraded using ``pip``. +is unexpectedly upgraded using :program:`pip`. On such systems, it is often better to use a virtual environment or a -per-user installation when installing packages with ``pip``. +per-user installation when installing packages with :program:`pip`. Pip not installed ----------------- -It is possible that ``pip`` does not get installed by default. One potential fix is:: +It is possible that :program:`pip` does not get installed by default. One potential fix is:: python -m ensurepip --default-pip -There are also additional resources for `installing pip. -`__ +There are also additional resources for `installing pip +`__. Installing binary extensions ---------------------------- -Python has typically relied heavily on source based distribution, with end +Python once relied heavily on source-based distribution, with end users being expected to compile extension modules from source as part of the installation process. -With the introduction of support for the binary ``wheel`` format, and the -ability to publish wheels for at least Windows and macOS through the -Python Package Index, this problem is expected to diminish over time, +With the introduction of the binary wheel format, and the +ability to publish wheels through PyPI, this problem is diminishing, as users are more regularly able to install pre-built extensions rather than needing to build them themselves. Some of the solutions for installing `scientific software `__ -that are not yet available as pre-built ``wheel`` files may also help with +that are not yet available as pre-built wheel files may also help with obtaining other binary extensions without needing to build them locally. .. seealso:: diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 5d916b30112d3c8..749e4543c5b8235 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -15,7 +15,7 @@ before the release in which the feature becomes standard. While these future statements are given additional special meaning by the Python compiler, they are still executed like any other import statement and -the :mod:`__future__` exists and is handled by the import system the same way +the :mod:`!__future__` exists and is handled by the import system the same way any other Python module would be. This design serves three purposes: * To avoid confusing existing tools that analyze import statements and expect to @@ -23,17 +23,17 @@ any other Python module would be. This design serves three purposes: * To document when incompatible changes were introduced, and when they will be --- or were --- made mandatory. This is a form of executable documentation, and - can be inspected programmatically via importing :mod:`__future__` and examining + can be inspected programmatically via importing :mod:`!__future__` and examining its contents. * To ensure that :ref:`future statements ` run under releases prior to - Python 2.1 at least yield runtime exceptions (the import of :mod:`__future__` + Python 2.1 at least yield runtime exceptions (the import of :mod:`!__future__` will fail, because there was no module of that name prior to 2.1). Module Contents --------------- -No feature description will ever be deleted from :mod:`__future__`. Since its +No feature description will ever be deleted from :mod:`!__future__`. Since its introduction in Python 2.1 the following features have found their way into the language using this mechanism: diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index ed29ac70035597c..13f463a1e95340e 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -36,11 +36,6 @@ This module defines the following constants and functions: This is now a synonym of the built-in :exc:`RuntimeError`. -.. data:: LockType - - This is the type of lock objects. - - .. function:: start_new_thread(function, args[, kwargs]) Start a new thread and return its identifier. The thread executes the @@ -120,13 +115,16 @@ This module defines the following constants and functions: Its value may be used to uniquely identify this particular thread system-wide (until the thread terminates, after which the value may be recycled by the OS). - .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD, GNU/kFreeBSD. + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD, GNU/kFreeBSD, Solaris. .. versionadded:: 3.8 .. versionchanged:: 3.13 Added support for GNU/kFreeBSD. + .. versionchanged:: 3.15 + Added support for Solaris. + .. function:: stack_size([size]) @@ -159,58 +157,66 @@ This module defines the following constants and functions: .. versionadded:: 3.2 -Lock objects have the following methods: +.. raw:: html + + + + + +.. class:: LockType -.. method:: lock.acquire(blocking=True, timeout=-1) + This is the type of lock objects. - Without any optional argument, this method acquires the lock unconditionally, if - necessary waiting until it is released by another thread (only one thread at a - time can acquire a lock --- that's their reason for existence). + Lock objects have the following methods: - If the *blocking* argument is present, the action depends on its - value: if it is false, the lock is only acquired if it can be acquired - immediately without waiting, while if it is true, the lock is acquired - unconditionally as above. + .. method:: acquire(blocking=True, timeout=-1) - If the floating-point *timeout* argument is present and positive, it - specifies the maximum wait time in seconds before returning. A negative - *timeout* argument specifies an unbounded wait. You cannot specify - a *timeout* if *blocking* is false. + Without any optional argument, this method acquires the lock unconditionally, if + necessary waiting until it is released by another thread (only one thread at a + time can acquire a lock --- that's their reason for existence). - The return value is ``True`` if the lock is acquired successfully, - ``False`` if not. + If the *blocking* argument is present, the action depends on its + value: if it is false, the lock is only acquired if it can be acquired + immediately without waiting, while if it is true, the lock is acquired + unconditionally as above. - .. versionchanged:: 3.2 - The *timeout* parameter is new. + If the floating-point *timeout* argument is present and positive, it + specifies the maximum wait time in seconds before returning. A negative + *timeout* argument specifies an unbounded wait. You cannot specify + a *timeout* if *blocking* is false. - .. versionchanged:: 3.2 - Lock acquires can now be interrupted by signals on POSIX. + The return value is ``True`` if the lock is acquired successfully, + ``False`` if not. - .. versionchanged:: 3.14 - Lock acquires can now be interrupted by signals on Windows. + .. versionchanged:: 3.2 + The *timeout* parameter is new. + .. versionchanged:: 3.2 + Lock acquires can now be interrupted by signals on POSIX. -.. method:: lock.release() + .. versionchanged:: 3.14 + Lock acquires can now be interrupted by signals on Windows. - Releases the lock. The lock must have been acquired earlier, but not - necessarily by the same thread. + .. method:: release() + Releases the lock. The lock must have been acquired earlier, but not + necessarily by the same thread. -.. method:: lock.locked() + .. method:: locked() - Return the status of the lock: ``True`` if it has been acquired by some thread, - ``False`` if not. + Return the status of the lock: ``True`` if it has been acquired by some thread, + ``False`` if not. -In addition to these methods, lock objects can also be used via the -:keyword:`with` statement, e.g.:: + In addition to these methods, lock objects can also be used via the + :keyword:`with` statement, e.g.:: - import _thread + import _thread - a_lock = _thread.allocate_lock() + a_lock = _thread.allocate_lock() - with a_lock: - print("a_lock is locked while this executes") + with a_lock: + print("a_lock is locked while this executes") **Caveats:** diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 49e541a9d9b1cb0..be25a94e7e94f6f 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -4,10 +4,6 @@ .. module:: abc :synopsis: Abstract base classes according to :pep:`3119`. -.. moduleauthor:: Guido van Rossum -.. sectionauthor:: Georg Brandl -.. much of the content adapted from docstrings - **Source code:** :source:`Lib/abc.py` -------------- @@ -170,17 +166,17 @@ The :mod:`!abc` module also provides the following decorator: or is derived from it. A class that has a metaclass derived from :class:`!ABCMeta` cannot be instantiated unless all of its abstract methods and properties are overridden. The abstract methods can be called using any - of the normal 'super' call mechanisms. :func:`!abstractmethod` may be used + of the normal 'super' call mechanisms. :deco:`!abstractmethod` may be used to declare abstract methods for properties and descriptors. Dynamically adding abstract methods to a class, or attempting to modify the abstraction status of a method or class once it is created, are only supported using the :func:`update_abstractmethods` function. The - :func:`!abstractmethod` only affects subclasses derived using regular + :deco:`!abstractmethod` only affects subclasses derived using regular inheritance; "virtual subclasses" registered with the ABC's :meth:`~ABCMeta.register` method are not affected. - When :func:`!abstractmethod` is applied in combination with other method + When :deco:`!abstractmethod` is applied in combination with other method descriptors, it should be applied as the innermost decorator, as shown in the following usage examples:: @@ -218,7 +214,7 @@ The :mod:`!abc` module also provides the following decorator: the descriptor must identify itself as abstract using :attr:`!__isabstractmethod__`. In general, this attribute should be ``True`` if any of the methods used to compose the descriptor are abstract. For - example, Python's built-in :class:`property` does the equivalent of:: + example, Python's built-in :deco:`property` does the equivalent of:: class Descriptor: ... @@ -241,14 +237,14 @@ The :mod:`!abc` module also supports the following legacy decorators: .. decorator:: abstractclassmethod .. versionadded:: 3.2 - .. deprecated:: 3.3 - It is now possible to use :class:`classmethod` with - :func:`abstractmethod`, making this decorator redundant. + .. deprecated-removed:: 3.3 3.21 + It is now possible to use :deco:`classmethod` with + :deco:`abstractmethod`, making this decorator redundant. - A subclass of the built-in :func:`classmethod`, indicating an abstract - classmethod. Otherwise it is similar to :func:`abstractmethod`. + A subclass of the built-in :class:`classmethod`, indicating an abstract + classmethod. Otherwise it is similar to :deco:`abstractmethod`. - This special case is deprecated, as the :func:`classmethod` decorator + This special case is deprecated, as the :deco:`classmethod` decorator is now correctly identified as abstract when applied to an abstract method:: @@ -262,14 +258,14 @@ The :mod:`!abc` module also supports the following legacy decorators: .. decorator:: abstractstaticmethod .. versionadded:: 3.2 - .. deprecated:: 3.3 - It is now possible to use :class:`staticmethod` with - :func:`abstractmethod`, making this decorator redundant. + .. deprecated-removed:: 3.3 3.21 + It is now possible to use :deco:`staticmethod` with + :deco:`abstractmethod`, making this decorator redundant. - A subclass of the built-in :func:`staticmethod`, indicating an abstract - staticmethod. Otherwise it is similar to :func:`abstractmethod`. + A subclass of the built-in :class:`staticmethod`, indicating an abstract + staticmethod. Otherwise it is similar to :deco:`abstractmethod`. - This special case is deprecated, as the :func:`staticmethod` decorator + This special case is deprecated, as the :deco:`staticmethod` decorator is now correctly identified as abstract when applied to an abstract method:: @@ -282,15 +278,15 @@ The :mod:`!abc` module also supports the following legacy decorators: .. decorator:: abstractproperty - .. deprecated:: 3.3 - It is now possible to use :class:`property`, :meth:`property.getter`, - :meth:`property.setter` and :meth:`property.deleter` with - :func:`abstractmethod`, making this decorator redundant. + .. deprecated-removed:: 3.3 3.21 + It is now possible to use :deco:`property`, :deco:`property.getter`, + :deco:`property.setter` and :deco:`property.deleter` with + :deco:`abstractmethod`, making this decorator redundant. - A subclass of the built-in :func:`property`, indicating an abstract + A subclass of the built-in :class:`property`, indicating an abstract property. - This special case is deprecated, as the :func:`property` decorator + This special case is deprecated, as the :deco:`property` decorator is now correctly identified as abstract when applied to an abstract method:: diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 981d89be7d58d68..af28fe0e2fde2f8 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -4,6 +4,7 @@ .. module:: annotationlib :synopsis: Functionality for introspecting annotations +.. versionadded:: 3.14 **Source code:** :source:`Lib/annotationlib.py` @@ -45,6 +46,10 @@ and :func:`call_annotate_function`, as well as the :func:`call_evaluate_function` function for working with :term:`evaluate functions `. +.. caution:: + + Most functionality in this module can execute arbitrary code; see + :ref:`the security section ` for more information. .. seealso:: @@ -335,14 +340,29 @@ Functions * VALUE: :attr:`!object.__annotations__` is tried first; if that does not exist, the :attr:`!object.__annotate__` function is called if it exists. + * FORWARDREF: If :attr:`!object.__annotations__` exists and can be evaluated successfully, it is used; otherwise, the :attr:`!object.__annotate__` function is called. If it does not exist either, :attr:`!object.__annotations__` is tried again and any error from accessing it is re-raised. + + * When calling :attr:`!object.__annotate__` it is first called with :attr:`~Format.FORWARDREF`. + If this is not implemented, it will then check if :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` + is supported and use that in the fake globals environment. + If neither of these formats are supported, it will fall back to using :attr:`~Format.VALUE`. + If :attr:`~Format.VALUE` fails, the error from this call will be raised. + * STRING: If :attr:`!object.__annotate__` exists, it is called first; otherwise, :attr:`!object.__annotations__` is used and stringified using :func:`annotations_to_string`. + * When calling :attr:`!object.__annotate__` it is first called with :attr:`~Format.STRING`. + If this is not implemented, it will then check if :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` + is supported and use that in the fake globals environment. + If neither of these formats are supported, it will fall back to using :attr:`~Format.VALUE` + with the result converted using :func:`annotations_to_string`. + If :attr:`~Format.VALUE` fails, the error from this call will be raised. + Returns a dict. :func:`!get_annotations` returns a new dict every time it's called; calling it twice on the same object will return two different but equivalent dicts. @@ -490,6 +510,81 @@ annotations from the class and puts them in a separate attribute: return typ +Creating a custom callable annotate function +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Custom :term:`annotate functions ` may be literal functions like those +automatically generated for functions, classes, and modules. Or, they may wish to utilise +the encapsulation provided by classes, in which case any :term:`callable` can be used as +an :term:`annotate function`. + +To provide the :attr:`~Format.VALUE`, :attr:`~Format.STRING`, or +:attr:`~Format.FORWARDREF` formats directly, an :term:`annotate function` must provide +the following attribute: + +* A callable ``__call__`` with signature ``__call__(format, /) -> dict``, that does not + raise a :exc:`NotImplementedError` when called with a supported format. + +To provide the :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` format, which is used to +automatically generate :attr:`~Format.STRING` or :attr:`~Format.FORWARDREF` if they are +not supported directly, :term:`annotate functions ` must provide the +following attributes: + +* A callable ``__call__`` with signature ``__call__(format, /) -> dict``, that does not + raise a :exc:`NotImplementedError` when called with + :attr:`~Format.VALUE_WITH_FAKE_GLOBALS`. +* A :ref:`code object ` ``__code__`` containing the compiled code for the + annotate function. +* Optional: A tuple of the function's positional defaults ``__kwdefaults__``, if the + function represented by ``__code__`` uses any positional defaults. +* Optional: A dict of the function's keyword defaults ``__defaults__``, if the function + represented by ``__code__`` uses any keyword defaults. +* Optional: All other :ref:`function attributes `. + +.. code-block:: python + + class Annotate: + called_formats = [] + + def __call__(self, format=None, /, *, _self=None): + # When called with fake globals, `_self` will be the + # actual self value, and `self` will be the format. + if _self is not None: + self, format = _self, self + + self.called_formats.append(format) + if format <= 2: # VALUE or VALUE_WITH_FAKE_GLOBALS + return {"x": MyType} + raise NotImplementedError + + __code__ = __call__.__code__ + __defaults__ = (None,) + __kwdefaults__ = property(lambda self: dict(_self=self)) + + __globals__ = {} + __builtins__ = {} + __closure__ = None + +This can then be called with: + +.. code-block:: pycon + + >>> from annotationlib import call_annotate_function, Format + >>> call_annotate_function(Annotate(), format=Format.STRING) + {'x': 'MyType'} + +Or used as the annotate function for an object: + +.. code-block:: pycon + + >>> from annotationlib import get_annotations, Format + >>> class C: + ... pass + >>> C.__annotate__ = Annotate() + >>> get_annotations(Annotate(), format=Format.STRING) + {'x': 'MyType'} + + Limitations of the ``STRING`` format ------------------------------------ @@ -603,3 +698,23 @@ Below are a few examples of the behavior with unsupported expressions: >>> def ifexp(x: 1 if y else 0): ... >>> get_annotations(ifexp, format=Format.STRING) {'x': '1'} + +.. _annotationlib-security: + +Security implications of introspecting annotations +-------------------------------------------------- + +Much of the functionality in this module involves executing code related to annotations, +which can then do arbitrary things. For example, +:func:`get_annotations` may call an arbitrary :term:`annotate function`, and +:meth:`ForwardRef.evaluate` may call :func:`eval` on an arbitrary string. Code contained +in an annotation might make arbitrary system calls, enter an infinite loop, or perform any +other operation. This is also true for any access of the :attr:`~object.__annotations__` attribute, +and for various functions in the :mod:`typing` module that work with annotations, such as +:func:`typing.get_type_hints`. + +Any security issue arising from this also applies immediately after importing +code that may contain untrusted annotations: importing code can always cause arbitrary operations +to be performed. However, it is unsafe to accept strings or other input from an untrusted source and +pass them to any of the APIs for introspecting annotations, for example by editing an +``__annotations__`` dictionary or directly creating a :class:`ForwardRef` object. diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 79e15994491eff8..e4a5f4d109b4992 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -4,16 +4,13 @@ .. module:: argparse :synopsis: Command-line option and argument parsing library. -.. moduleauthor:: Steven Bethard -.. sectionauthor:: Steven Bethard - .. versionadded:: 3.2 **Source code:** :source:`Lib/argparse.py` .. note:: - While :mod:`argparse` is the default recommended standard library module + While :mod:`!argparse` is the default recommended standard library module for implementing basic command line applications, authors with more exacting requirements for exactly how their command line applications behave may find it doesn't provide the necessary level of control. @@ -74,7 +71,7 @@ ArgumentParser objects prefix_chars='-', fromfile_prefix_chars=None, \ argument_default=None, conflict_handler='error', \ add_help=True, allow_abbrev=True, exit_on_error=True, \ - *, suggest_on_error=False, color=True) + *, suggest_on_error=True, color=True) Create a new :class:`ArgumentParser` object. All parameters should be passed as keyword arguments. Each parameter has its own more detailed description @@ -117,7 +114,7 @@ ArgumentParser objects error info when an error occurs. (default: ``True``) * suggest_on_error_ - Enables suggestions for mistyped argument choices - and subparser names (default: ``False``) + and subparser names (default: ``True``) * color_ - Allow color output (default: ``True``) @@ -134,6 +131,9 @@ ArgumentParser objects .. versionchanged:: 3.14 *suggest_on_error* and *color* parameters were added. + .. versionchanged:: 3.15 + *suggest_on_error* default changed to ``True``. + The following sections describe how each of these are used. @@ -442,9 +442,8 @@ is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. .. note:: - Empty lines are treated as empty strings (``''``), which are allowed as values but - not as arguments. Empty lines that are read as arguments will result in an - "unrecognized arguments" error. + Each line is treated as a single argument, so an empty line is read as an + empty string (``''``). :class:`ArgumentParser` uses :term:`filesystem encoding and error handler` to read the file containing arguments. @@ -596,32 +595,23 @@ suggest_on_error ^^^^^^^^^^^^^^^^ By default, when a user passes an invalid argument choice or subparser name, -:class:`ArgumentParser` will exit with error info and list the permissible -argument choices (if specified) or subparser names as part of the error message. - -If the user would like to enable suggestions for mistyped argument choices and -subparser names, the feature can be enabled by setting ``suggest_on_error`` to -``True``. Note that this only applies for arguments when the choices specified -are strings:: - - >>> parser = argparse.ArgumentParser(description='Process some integers.', - suggest_on_error=True) - >>> parser.add_argument('--action', choices=['sum', 'max']) - >>> parser.add_argument('integers', metavar='N', type=int, nargs='+', - ... help='an integer for the accumulator') - >>> parser.parse_args(['--action', 'sumn', 1, 2, 3]) - tester.py: error: argument --action: invalid choice: 'sumn', maybe you meant 'sum'? (choose from 'sum', 'max') +:class:`ArgumentParser` will exit with error info and provide suggestions for +mistyped arguments. The error message will list the permissible argument +choices (if specified) or subparser names, along with a "maybe you meant" +suggestion if a close match is found. Note that this only applies for arguments +when the choices specified are strings:: -If you're writing code that needs to be compatible with older Python versions -and want to opportunistically use ``suggest_on_error`` when it's available, you -can set it as an attribute after initializing the parser instead of using the -keyword argument:: + >>> parser = argparse.ArgumentParser(suggest_on_error=True) + >>> parser.add_argument('--action', choices=['debug', 'dryrun']) + >>> parser.parse_args(['--action', 'debugg']) + usage: tester.py [-h] [--action {debug,dryrun}] + tester.py: error: argument --action: invalid choice: 'debugg', maybe you meant 'debug'? (choose from debug, dryrun) - >>> parser = argparse.ArgumentParser(description='Process some integers.') - >>> parser.suggest_on_error = True +You can disable suggestions by setting ``suggest_on_error`` to ``False``. .. versionadded:: 3.14 - +.. versionchanged:: 3.15 + Changed default value of ``suggest_on_error`` from ``False`` to ``True``. color ^^^^^ @@ -639,8 +629,31 @@ by setting ``color`` to ``False``:: ... help='an integer for the accumulator') >>> parser.parse_args(['--help']) +Note that when ``color=True``, colored output depends on both environment +variables and terminal capabilities. However, if ``color=False``, colored +output is always disabled, even if environment variables like ``FORCE_COLOR`` +are set. + .. versionadded:: 3.14 +To highlight inline code in your description, epilog, or argument ``help`` +text, you can use single or double backticks:: + + >>> parser = argparse.ArgumentParser( + ... formatter_class=argparse.RawDescriptionHelpFormatter, + ... description='Run ``python -m myapp`` to start.', + ... epilog='''Examples: + ... `python -m myapp --verbose` + ... ``python -m myapp --config settings.json`` + ... ''') + >>> parser.add_argument('--foo', help='set the `foo` value') + +When colors are enabled, the text inside backticks will be displayed in a +distinct color to help examples stand out. When colors are disabled, backticks +are preserved as-is, which is readable in plain text. + +.. versionadded:: 3.15 + The add_argument() method ------------------------- @@ -681,6 +694,8 @@ The add_argument() method * deprecated_ - Whether or not use of the argument is deprecated. + The method returns an :class:`Action` object representing the argument. + The following sections describe how each of these are used. @@ -722,13 +737,13 @@ By default, :mod:`!argparse` automatically handles the internal naming and display names of arguments, simplifying the process without requiring additional configuration. As such, you do not need to specify the dest_ and metavar_ parameters. -The dest_ parameter defaults to the argument name with underscores ``_`` -replacing hyphens ``-`` . The metavar_ parameter defaults to the -upper-cased name. For example:: +For optional arguments, the dest_ parameter defaults to the argument name, with +underscores ``_`` replacing hyphens ``-``. The metavar_ parameter defaults to +the upper-cased name. For example:: >>> parser = argparse.ArgumentParser(prog='PROG') >>> parser.add_argument('--foo-bar') - >>> parser.parse_args(['--foo-bar', 'FOO-BAR'] + >>> parser.parse_args(['--foo-bar', 'FOO-BAR']) Namespace(foo_bar='FOO-BAR') >>> parser.print_help() usage: [-h] [--foo-bar FOO-BAR] @@ -763,9 +778,9 @@ how the command-line arguments should be handled. The supplied actions are: Namespace(foo=42) * ``'store_true'`` and ``'store_false'`` - These are special cases of - ``'store_const'`` used for storing the values ``True`` and ``False`` - respectively. In addition, they create default values of ``False`` and - ``True`` respectively:: + ``'store_const'`` that respectively store the values ``True`` and ``False`` + with default values of ``False`` and + ``True``:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', action='store_true') @@ -774,19 +789,19 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args('--foo --bar'.split()) Namespace(foo=True, bar=False, baz=True) -* ``'append'`` - This stores a list, and appends each argument value to the - list. It is useful to allow an option to be specified multiple times. - If the default value is non-empty, the default elements will be present - in the parsed value for the option, with any values from the - command line appended after those default values. Example usage:: +* ``'append'`` - This appends each argument value to a list. + It is useful for allowing an option to be specified multiple times. + If the default value is a non-empty list, the parsed value will start + with the default list's elements and any values from the command line + will be appended after those default values. Example usage:: >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action='append') + >>> parser.add_argument('--foo', action='append', default=['0']) >>> parser.parse_args('--foo 1 --foo 2'.split()) - Namespace(foo=['1', '2']) + Namespace(foo=['0', '1', '2']) -* ``'append_const'`` - This stores a list, and appends the value specified by - the const_ keyword argument to the list; note that the const_ keyword +* ``'append_const'`` - This appends the value specified by + the const_ keyword argument to a list; note that the const_ keyword argument defaults to ``None``. The ``'append_const'`` action is typically useful when multiple arguments need to store constants to the same list. For example:: @@ -797,8 +812,8 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args('--str --int'.split()) Namespace(types=[, ]) -* ``'extend'`` - This stores a list and appends each item from the multi-value - argument list to it. +* ``'extend'`` - This appends each item from a multi-value + argument to a list. The ``'extend'`` action is typically used with the nargs_ keyword argument value ``'+'`` or ``'*'``. Note that when nargs_ is ``None`` (the default) or ``'?'``, each @@ -812,7 +827,7 @@ how the command-line arguments should be handled. The supplied actions are: .. versionadded:: 3.8 -* ``'count'`` - This counts the number of times a keyword argument occurs. For +* ``'count'`` - This counts the number of times an argument occurs. For example, this is useful for increasing verbosity levels:: >>> parser = argparse.ArgumentParser() @@ -981,8 +996,8 @@ the various :class:`ArgumentParser` actions. The two most common uses of it are (like ``-f`` or ``--foo``) and ``nargs='?'``. This creates an optional argument that can be followed by zero or one command-line arguments. When parsing the command line, if the option string is encountered with no - command-line argument following it, the value of ``const`` will be assumed to - be ``None`` instead. See the nargs_ description for examples. + command-line argument following it, the value from ``const`` will be used. + See the nargs_ description for examples. .. versionchanged:: 3.11 ``const=None`` by default, including when ``action='append_const'`` or @@ -1036,6 +1051,10 @@ is used when no command-line argument was present:: >>> parser.parse_args([]) Namespace(foo=42) +Because ``nargs='*'`` gathers any supplied values into a list, an absent +positional argument yields an empty list (``[]``). Only a non-``None`` +*default* overrides this (so ``default=None`` still gives ``[]``). + For required_ arguments, the ``default`` value is ignored. For example, this applies to positional arguments with nargs_ values other than ``?`` or ``*``, or optional arguments marked as ``required=True``. @@ -1099,7 +1118,15 @@ User defined functions can be used as well: The :func:`bool` function is not recommended as a type converter. All it does is convert empty strings to ``False`` and non-empty strings to ``True``. -This is usually not what is desired. +This is usually not what is desired:: + + >>> parser = argparse.ArgumentParser() + >>> _ = parser.add_argument('--verbose', type=bool) + >>> parser.parse_args(['--verbose', 'False']) + Namespace(verbose=True) + +See :class:`BooleanOptionalAction` or ``action='store_true'`` for common +alternatives. In general, the ``type`` keyword is a convenience that should only be used for simple conversions that can only raise one of the three supported exceptions. @@ -1142,16 +1169,21 @@ if the argument was not one of the acceptable values:: game.py: error: argument move: invalid choice: 'fire' (choose from 'rock', 'paper', 'scissors') -Note that inclusion in the *choices* sequence is checked after any type_ -conversions have been performed, so the type of the objects in the *choices* -sequence should match the type_ specified. - Any sequence can be passed as the *choices* value, so :class:`list` objects, :class:`tuple` objects, and custom sequences are all supported. Use of :class:`enum.Enum` is not recommended because it is difficult to control its appearance in usage, help, and error messages. +Note that *choices* are checked after any type_ +conversions have been performed, so objects in *choices* +should match the type_ specified. This can make *choices* +appear unfamiliar in usage, help, or error messages. + +To keep *choices* user-friendly, consider a custom type wrapper that +converts and formats values, or omit type_ and handle conversion in +your application code. + Formatted choices override the default *metavar* which is normally derived from *dest*. This is usually what you want because the user never sees the *dest* parameter. If this display isn't desirable (perhaps because there are @@ -1313,8 +1345,12 @@ attribute is determined by the ``dest`` keyword argument of For optional argument actions, the value of ``dest`` is normally inferred from the option strings. :class:`ArgumentParser` generates the value of ``dest`` by -taking the first long option string and stripping away the initial ``--`` -string. If no long option strings were supplied, ``dest`` will be derived from +taking the first double-dash long option string and stripping away the initial +``-`` characters. +If no double-dash long option strings were supplied, ``dest`` will be derived +from the first single-dash long option string by stripping the initial ``-`` +character. +If no long option strings were supplied, ``dest`` will be derived from the first short option string by stripping the initial ``-`` character. Any internal ``-`` characters will be converted to ``_`` characters to make sure the string is a valid attribute name. The examples below illustrate this @@ -1322,11 +1358,12 @@ behavior:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('-f', '--foo-bar', '--foo') + >>> parser.add_argument('-q', '-quz') >>> parser.add_argument('-x', '-y') - >>> parser.parse_args('-f 1 -x 2'.split()) - Namespace(foo_bar='1', x='2') - >>> parser.parse_args('--foo 1 -y 2'.split()) - Namespace(foo_bar='1', x='2') + >>> parser.parse_args('-f 1 -q 2 -x 3'.split()) + Namespace(foo_bar='1', quz='2', x='3') + >>> parser.parse_args('--foo 1 -quz 2 -y 3'.split()) + Namespace(foo_bar='1', quz='2', x='2') ``dest`` allows a custom attribute name to be provided:: @@ -1335,6 +1372,14 @@ behavior:: >>> parser.parse_args('--foo XXX'.split()) Namespace(bar='XXX') +Multiple arguments may share the same ``dest``. By default, the value from the +last such argument given on the command line wins. Use ``action='append'`` to +collect values from all of them into a list instead. For conflicting *option +strings* rather than ``dest`` names, see conflict_handler_. + +.. versionchanged:: 3.15 + Single-dash long option now takes precedence over short options. + .. _deprecated: @@ -1428,8 +1473,18 @@ this API may be passed as the ``action`` parameter to >>> parser.parse_args(['--no-foo']) Namespace(foo=False) + Single-dash long options are also supported. + For example, negative option ``-nofoo`` is automatically added for + positive option ``-foo``. + But no additional options are added for short options such as ``-f``. + .. versionadded:: 3.9 + .. versionchanged:: 3.15 + Added support for single-dash options. + + Added support for alternate prefix_chars_. + The parse_args() method ----------------------- @@ -1652,7 +1707,7 @@ The Namespace object Other utilities --------------- -Sub-commands +Subcommands ^^^^^^^^^^^^ .. method:: ArgumentParser.add_subparsers(*, [title], [description], [prog], \ @@ -1681,7 +1736,7 @@ Sub-commands * *description* - description for the sub-parser group in help output, by default ``None`` - * *prog* - usage information that will be displayed with sub-command help, + * *prog* - usage information that will be displayed with subcommand help, by default the name of the program and any positional arguments before the subparser argument @@ -1691,7 +1746,7 @@ Sub-commands * action_ - the basic type of action to be taken when this argument is encountered at the command line - * dest_ - name of the attribute under which sub-command name will be + * dest_ - name of the attribute under which subcommand name will be stored; by default ``None`` and no value is stored * required_ - Whether or not a subcommand must be provided, by default @@ -1723,13 +1778,18 @@ Sub-commands >>> parser.parse_args(['--foo', 'b', '--baz', 'Z']) Namespace(baz='Z', foo=True) - Note that the object returned by :meth:`parse_args` will only contain + Note that the object returned by :meth:`~ArgumentParser.parse_args` will only contain attributes for the main parser and the subparser that was selected by the command line (and not any other subparsers). So in the example above, when the ``a`` command is specified, only the ``foo`` and ``bar`` attributes are present, and when the ``b`` command is specified, only the ``foo`` and ``baz`` attributes are present. + If a subparser defines an argument with the same ``dest`` as the parent + parser, the two share a single namespace attribute, so the parent's value + won't be retained. Users should give them distinct ``dest`` values to + keep both. + Similarly, when a help message is requested from a subparser, only the help for that particular parser will be printed. The help message will not include parent parser or sibling parser messages. (A help message for each @@ -1766,7 +1826,7 @@ Sub-commands -h, --help show this help message and exit --baz {X,Y,Z} baz help - The :meth:`add_subparsers` method also supports ``title`` and ``description`` + The :meth:`~ArgumentParser.add_subparsers` method also supports ``title`` and ``description`` keyword arguments. When either is present, the subparser's commands will appear in their own group in the help output. For example:: @@ -1787,34 +1847,8 @@ Sub-commands {foo,bar} additional help - Furthermore, :meth:`~_SubParsersAction.add_parser` supports an additional - *aliases* argument, - which allows multiple strings to refer to the same subparser. This example, - like ``svn``, aliases ``co`` as a shorthand for ``checkout``:: - - >>> parser = argparse.ArgumentParser() - >>> subparsers = parser.add_subparsers() - >>> checkout = subparsers.add_parser('checkout', aliases=['co']) - >>> checkout.add_argument('foo') - >>> parser.parse_args(['co', 'bar']) - Namespace(foo='bar') - - :meth:`~_SubParsersAction.add_parser` supports also an additional - *deprecated* argument, which allows to deprecate the subparser. - - >>> import argparse - >>> parser = argparse.ArgumentParser(prog='chicken.py') - >>> subparsers = parser.add_subparsers() - >>> run = subparsers.add_parser('run') - >>> fly = subparsers.add_parser('fly', deprecated=True) - >>> parser.parse_args(['fly']) # doctest: +SKIP - chicken.py: warning: command 'fly' is deprecated - Namespace() - - .. versionadded:: 3.13 - One particularly effective way of handling subcommands is to combine the use - of the :meth:`add_subparsers` method with calls to :meth:`set_defaults` so + of the :meth:`~ArgumentParser.add_subparsers` method with calls to :meth:`~ArgumentParser.set_defaults` so that each subparser knows which Python function it should execute. For example:: @@ -1850,12 +1884,12 @@ Sub-commands >>> args.func(args) ((XYZYX)) - This way, you can let :meth:`parse_args` do the job of calling the + This way, you can let :meth:`~ArgumentParser.parse_args` do the job of calling the appropriate function after argument parsing is complete. Associating functions with actions like this is typically the easiest way to handle the different actions for each of your subparsers. However, if it is necessary to check the name of the subparser that was invoked, the ``dest`` keyword - argument to the :meth:`add_subparsers` call will work:: + argument to the :meth:`~ArgumentParser.add_subparsers` call will work:: >>> parser = argparse.ArgumentParser() >>> subparsers = parser.add_subparsers(dest='subparser_name') @@ -1874,6 +1908,43 @@ Sub-commands the main parser. +.. method:: _SubParsersAction.add_parser(name, *, help=None, aliases=None, \ + deprecated=False, **kwargs) + + Create and return a new :class:`ArgumentParser` object for the + subcommand *name*. + + The *name* argument is the name of the sub-command. + + The *help* argument provides a short description for this sub-command. + + The *aliases* argument allows providing alternative names for this + sub-command. For example:: + + >>> parser = argparse.ArgumentParser() + >>> subparsers = parser.add_subparsers() + >>> checkout = subparsers.add_parser('checkout', aliases=['co']) + >>> checkout.add_argument('foo') + >>> parser.parse_args(['co', 'bar']) + Namespace(foo='bar') + + The *deprecated* argument, if ``True``, marks the sub-command as + deprecated and will issue a warning when used. For example:: + + >>> parser = argparse.ArgumentParser(prog='chicken.py') + >>> subparsers = parser.add_subparsers() + >>> fly = subparsers.add_parser('fly', deprecated=True) + >>> args = parser.parse_args(['fly']) + chicken.py: warning: command 'fly' is deprecated + Namespace() + + All other keyword arguments are passed directly to the + :class:`!ArgumentParser` constructor. + + .. versionadded:: 3.13 + Added the *deprecated* parameter. + + FileType objects ^^^^^^^^^^^^^^^^ @@ -1909,7 +1980,7 @@ FileType objects run and then use the :keyword:`with`-statement to manage the files. .. versionchanged:: 3.4 - Added the *encodings* and *errors* parameters. + Added the *encoding* and *errors* parameters. .. deprecated:: 3.14 @@ -1971,6 +2042,9 @@ Argument groups Note that any arguments not in your user-defined groups will end up back in the usual "positional arguments" and "optional arguments" sections. + Within each argument group, arguments are displayed in help output in the + order in which they are added. + .. deprecated-removed:: 3.11 3.14 Calling :meth:`add_argument_group` on an argument group now raises an exception. This nesting was never supported, often failed to work @@ -2061,7 +2135,9 @@ Parser defaults >>> parser.parse_args(['736']) Namespace(bar=42, baz='badger', foo=736) - Note that parser-level defaults always override argument-level defaults:: + Note that defaults can be set at both the parser level using :meth:`set_defaults` + and at the argument level using :meth:`add_argument`. If both are called for the + same argument, the last default set for an argument is used:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', default='bar') @@ -2169,6 +2245,9 @@ Customizing file parsing def convert_arg_line_to_args(self, arg_line): return arg_line.split() + Note that with this override an argument can no longer contain spaces, since + each space-separated word becomes a separate argument. + Exiting methods ^^^^^^^^^^^^^^^ diff --git a/Doc/library/array.rst b/Doc/library/array.rst index e0b1eb89cf6c05a..c688d54318e707f 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -9,10 +9,10 @@ -------------- This module defines an object type which can compactly represent an array of -basic values: characters, integers, floating-point numbers. Arrays are sequence +basic values: characters, integers, floating-point numbers, complex numbers. Arrays are mutable :term:`sequence` types and behave very much like lists, except that the type of objects stored in them is constrained. The type is specified at object creation time by using a -:dfn:`type code`, which is a single character. The following type codes are +:dfn:`type code`. The following type codes are defined: +-----------+--------------------+-------------------+-----------------------+-------+ @@ -22,9 +22,7 @@ defined: +-----------+--------------------+-------------------+-----------------------+-------+ | ``'B'`` | unsigned char | int | 1 | | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'u'`` | wchar_t | Unicode character | 2 | \(1) | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'w'`` | Py_UCS4 | Unicode character | 4 | | +| ``'w'`` | Py_UCS4 | Unicode character | 4 | \(1) | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'h'`` | signed short | int | 2 | | +-----------+--------------------+-------------------+-----------------------+-------+ @@ -42,23 +40,47 @@ defined: +-----------+--------------------+-------------------+-----------------------+-------+ | ``'Q'`` | unsigned long long | int | 8 | | +-----------+--------------------+-------------------+-----------------------+-------+ +| ``'e'`` | _Float16 | float | 2 | \(2) | ++-----------+--------------------+-------------------+-----------------------+-------+ | ``'f'`` | float | float | 4 | | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'d'`` | double | float | 8 | | +-----------+--------------------+-------------------+-----------------------+-------+ +| ``'Zf'`` | float complex | complex | 8 | \(3) | ++-----------+--------------------+-------------------+-----------------------+-------+ +| ``'Zd'`` | double complex | complex | 16 | \(3) | ++-----------+--------------------+-------------------+-----------------------+-------+ + Notes: (1) - It can be 16 bits or 32 bits depending on the platform. + .. versionadded:: 3.13 + +(2) + The IEEE 754 binary16 "half precision" type was introduced in the 2008 + revision of the `IEEE 754 standard `_. + This type is not widely supported by C compilers. It's available + as :c:expr:`_Float16` type, if the compiler supports the Annex H + of the C23 standard. - .. versionchanged:: 3.9 - ``array('u')`` now uses :c:type:`wchar_t` as C type instead of deprecated - ``Py_UNICODE``. This change doesn't affect its behavior because - ``Py_UNICODE`` is alias of :c:type:`wchar_t` since Python 3.3. + .. versionadded:: 3.15 - .. deprecated-removed:: 3.3 3.16 - Please migrate to ``'w'`` typecode. +(3) + Complex types (``Zf`` and ``Zd``) are available unconditionally, + regardless on support for complex types (the Annex G of the C11 standard) + by the C compiler. + As specified in the C11 standard, each complex type is represented by a + two-element C array containing, respectively, the real and imaginary parts. + + .. versionadded:: 3.15 + +.. seealso:: + + The :ref:`ctypes ` and + :ref:`struct ` modules, + as well as third-party modules like `numpy `__, + use similar -- but slightly different -- type codes. The actual representation of values is determined by the machine architecture @@ -70,7 +92,10 @@ The module defines the following item: .. data:: typecodes - A string with all available type codes. + A tuple with all available type codes. + + .. versionchanged:: 3.15 + The type changed from :class:`str` to :class:`tuple`. The module defines the following type: @@ -90,12 +115,14 @@ The module defines the following type: otherwise, the initializer's iterator is passed to the :meth:`extend` method to add initial items to the array. - Array objects support the ordinary sequence operations of indexing, slicing, + Array objects support the ordinary :ref:`mutable ` :term:`sequence` operations of indexing, slicing, concatenation, and multiplication. When using slice assignment, the assigned value must be an array object with the same type code; in all other cases, :exc:`TypeError` is raised. Array objects also implement the buffer interface, and may be used wherever :term:`bytes-like objects ` are supported. + Arrays are :ref:`generic ` over the type of their contents. + .. audit-event:: array.__new__ typecode,initializer array.array @@ -109,9 +136,9 @@ The module defines the following type: The length in bytes of one array item in the internal representation. - .. method:: append(x) + .. method:: append(value, /) - Append a new item with value *x* to the end of the array. + Append a new item with the specified value to the end of the array. .. method:: buffer_info() @@ -136,17 +163,18 @@ The module defines the following type: .. method:: byteswap() "Byteswap" all items of the array. This is only supported for values which are - 1, 2, 4, or 8 bytes in size; for other types of values, :exc:`RuntimeError` is + 1, 2, 4, 8 or 16 bytes in size; for other types of values, :exc:`RuntimeError` is raised. It is useful when reading data from a file written on a machine with a - different byte order. + different byte order. Note, that for complex types the order of + components (the real part, followed by imaginary part) is preserved. - .. method:: count(x) + .. method:: count(value, /) - Return the number of occurrences of *x* in the array. + Return the number of occurrences of *value* in the array. - .. method:: extend(iterable) + .. method:: extend(iterable, /) Append items from *iterable* to the end of the array. If *iterable* is another array, it must have *exactly* the same type code; if not, :exc:`TypeError` will @@ -154,7 +182,7 @@ The module defines the following type: must be the right type to be appended to the array. - .. method:: frombytes(buffer) + .. method:: frombytes(buffer, /) Appends items from the :term:`bytes-like object`, interpreting its content as an array of machine values (as if it had been read @@ -164,7 +192,7 @@ The module defines the following type: :meth:`!fromstring` is renamed to :meth:`frombytes` for clarity. - .. method:: fromfile(f, n) + .. method:: fromfile(f, n, /) Read *n* items (as machine values) from the :term:`file object` *f* and append them to the end of the array. If less than *n* items are available, @@ -172,47 +200,47 @@ The module defines the following type: inserted into the array. - .. method:: fromlist(list) + .. method:: fromlist(list, /) Append items from the list. This is equivalent to ``for x in list: a.append(x)`` except that if there is a type error, the array is unchanged. - .. method:: fromunicode(s) + .. method:: fromunicode(ustr, /) Extends this array with data from the given Unicode string. - The array must have type code ``'u'`` or ``'w'``; otherwise a :exc:`ValueError` is raised. + The array must have type code ``'w'``; otherwise a :exc:`ValueError` is raised. Use ``array.frombytes(unicodestring.encode(enc))`` to append Unicode data to an array of some other type. - .. method:: index(x[, start[, stop]]) + .. method:: index(value[, start[, stop]]) Return the smallest *i* such that *i* is the index of the first occurrence of - *x* in the array. The optional arguments *start* and *stop* can be - specified to search for *x* within a subsection of the array. Raise - :exc:`ValueError` if *x* is not found. + *value* in the array. The optional arguments *start* and *stop* can be + specified to search for *value* within a subsection of the array. Raise + :exc:`ValueError` if *value* is not found. .. versionchanged:: 3.10 Added optional *start* and *stop* parameters. - .. method:: insert(i, x) + .. method:: insert(index, value, /) - Insert a new item with value *x* in the array before position *i*. Negative + Insert a new item *value* in the array before position *index*. Negative values are treated as being relative to the end of the array. - .. method:: pop([i]) + .. method:: pop(index=-1, /) Removes the item with the index *i* from the array and returns it. The optional argument defaults to ``-1``, so that by default the last item is removed and returned. - .. method:: remove(x) + .. method:: remove(value, /) - Remove the first occurrence of *x* from the array. + Remove the first occurrence of *value* from the array. .. method:: clear() @@ -237,7 +265,7 @@ The module defines the following type: :meth:`!tostring` is renamed to :meth:`tobytes` for clarity. - .. method:: tofile(f) + .. method:: tofile(f, /) Write all items (as machine values) to the :term:`file object` *f*. @@ -249,7 +277,7 @@ The module defines the following type: .. method:: tounicode() - Convert the array to a Unicode string. The array must have a type ``'u'`` or ``'w'``; + Convert the array to a Unicode string. The array must have a type ``'w'``; otherwise a :exc:`ValueError` is raised. Use ``array.tobytes().decode(enc)`` to obtain a Unicode string from an array of some other type. @@ -257,7 +285,7 @@ The module defines the following type: The string representation of array objects has the form ``array(typecode, initializer)``. The *initializer* is omitted if the array is empty, otherwise it is -a Unicode string if the *typecode* is ``'u'`` or ``'w'``, otherwise it is +a Unicode string if the *typecode* is ``'w'``, otherwise it is a list of numbers. The string representation is guaranteed to be able to be converted back to an array with the same type and value using :func:`eval`, so long as the @@ -279,3 +307,5 @@ Examples:: `NumPy `_ The NumPy package defines another array type. + +.. _ieee 754 standard: https://en.wikipedia.org/wiki/IEEE_754-2008_revision diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index b24459b5c6346f7..4809fdb42bf3d77 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -4,9 +4,6 @@ .. module:: ast :synopsis: Abstract Syntax Tree classes and manipulation. -.. sectionauthor:: Martin v. Löwis -.. sectionauthor:: Georg Brandl - .. testsetup:: import ast @@ -15,7 +12,7 @@ -------------- -The :mod:`ast` module helps Python applications to process trees of the Python +The :mod:`!ast` module helps Python applications to process trees of the Python abstract syntax grammar. The abstract syntax itself might change with each Python release; this module helps to find out programmatically what the current grammar looks like. @@ -38,15 +35,17 @@ The abstract grammar is currently defined as follows: :language: asdl +.. _ast_nodes: + Node classes ------------ .. class:: AST - This is the base of all AST node classes. The actual node classes are + This is the abstract base of all AST node classes. The actual node classes are derived from the :file:`Parser/Python.asdl` file, which is reproduced :ref:`above `. They are defined in the :mod:`!_ast` C - module and re-exported in :mod:`ast`. + module and re-exported in :mod:`!ast`. There is one class defined for each left-hand side symbol in the abstract grammar (for example, :class:`ast.stmt` or :class:`ast.expr`). In addition, @@ -134,17 +133,26 @@ Node classes Simple indices are represented by their value, extended slices are represented as tuples. +.. versionchanged:: 3.13 + + AST node constructors were changed to provide sensible defaults for omitted + fields: optional fields now default to ``None``, list fields default to an + empty list, and fields of type :class:`!ast.expr_context` default to + :class:`Load() `. Previously, omitted attributes would not exist on constructed + nodes (accessing them raised :exc:`AttributeError`). + .. versionchanged:: 3.14 The :meth:`~object.__repr__` output of :class:`~ast.AST` nodes includes the values of the node fields. -.. deprecated:: 3.8 +.. deprecated-removed:: 3.8 3.14 - Old classes :class:`!ast.Num`, :class:`!ast.Str`, :class:`!ast.Bytes`, - :class:`!ast.NameConstant` and :class:`!ast.Ellipsis` are still available, - but they will be removed in future Python releases. In the meantime, - instantiating them will return an instance of a different class. + Previous versions of Python provided the AST classes :class:`!ast.Num`, + :class:`!ast.Str`, :class:`!ast.Bytes`, :class:`!ast.NameConstant` and + :class:`!ast.Ellipsis`, which were deprecated in Python 3.8. These classes + were removed in Python 3.14, and their functionality has been replaced with + :class:`ast.Constant`. .. deprecated:: 3.9 @@ -158,8 +166,16 @@ Node classes Previous versions of Python allowed the creation of AST nodes that were missing required fields. Similarly, AST node constructors allowed arbitrary keyword arguments that were set as attributes of the AST node, even if they did not - match any of the fields of the AST node. This behavior is deprecated and will - be removed in Python 3.15. + match any of the fields of the AST node. These cases now raise a :exc:`TypeError`. + +.. deprecated-removed:: 3.15 3.20 + + In the :ref:`grammar above `, the AST node classes that + correspond to production rules with variants (aka "sums") are abstract + classes. Previous versions of Python allowed for the creation of direct + instances of these abstract node classes. This behavior is deprecated and + will be removed in Python 3.20. + .. note:: The descriptions of the specific node classes displayed here @@ -264,18 +280,25 @@ Root nodes Literals ^^^^^^^^ -.. class:: Constant(value) +.. class:: Constant(value, kind) A constant value. The ``value`` attribute of the ``Constant`` literal contains the Python object it represents. The values represented can be instances of :class:`str`, :class:`bytes`, :class:`int`, :class:`float`, :class:`complex`, and :class:`bool`, and the constants :data:`None` and :data:`Ellipsis`. + The ``kind`` attribute is an optional string. For string literals with a + ``u`` prefix, ``kind`` is set to ``'u'``. For all other + constants, ``kind`` is ``None``. + .. doctest:: >>> print(ast.dump(ast.parse('123', mode='eval'), indent=4)) Expression( body=Constant(value=123)) + >>> print(ast.dump(ast.parse("u'hello'", mode='eval'), indent=4)) + Expression( + body=Constant(value='hello', kind='u')) .. class:: FormattedValue(value, conversion, format_spec) @@ -289,9 +312,9 @@ Literals * ``conversion`` is an integer: * -1: no formatting - * 115 (``ord('s')``): ``!s`` string formatting - * 114 (``ord('r')``): ``!r`` repr formatting - * 97 (``ord('a')``): ``!a`` ASCII formatting + * 97 (``ord('a')``): ``!a`` :func:`ASCII ` formatting + * 114 (``ord('r')``): ``!r`` :func:`repr` formatting + * 115 (``ord('s')``): ``!s`` :func:`string ` formatting * ``format_spec`` is a :class:`JoinedStr` node representing the formatting of the value, or ``None`` if no format was specified. Both @@ -325,14 +348,18 @@ Literals Constant(value='.3')]))])) -.. class:: TemplateStr(values) +.. class:: TemplateStr(values, /) - A t-string, comprising a series of :class:`Interpolation` and :class:`Constant` - nodes. + .. versionadded:: 3.14 + + Node representing a template string literal, comprising a series of + :class:`Interpolation` and :class:`Constant` nodes. + These nodes may be any order, and do not need to be interleaved. .. doctest:: - >>> print(ast.dump(ast.parse('t"{name} finished {place:ordinal}"', mode='eval'), indent=4)) + >>> expr = ast.parse('t"{name} finished {place:ordinal}"', mode='eval') + >>> print(ast.dump(expr, indent=4)) Expression( body=TemplateStr( values=[ @@ -349,28 +376,33 @@ Literals values=[ Constant(value='ordinal')]))])) - .. versionadded:: 3.14 - +.. class:: Interpolation(value, str, conversion, format_spec=None) -.. class:: Interpolation(value, str, conversion, format_spec) + .. versionadded:: 3.14 - Node representing a single interpolation field in a t-string. + Node representing a single interpolation field in a template string literal. * ``value`` is any expression node (such as a literal, a variable, or a function call). + This has the same meaning as ``FormattedValue.value``. * ``str`` is a constant containing the text of the interpolation expression. + + If ``str`` is set to ``None``, then ``value`` is used to generate code + when calling :func:`ast.unparse`. This no longer guarantees that the + generated code is identical to the original and is intended for code + generation. * ``conversion`` is an integer: * -1: no conversion - * 115: ``!s`` string conversion - * 114: ``!r`` repr conversion - * 97: ``!a`` ascii conversion + * 97 (``ord('a')``): ``!a`` :func:`ASCII ` conversion + * 114 (``ord('r')``): ``!r`` :func:`repr` conversion + * 115 (``ord('s')``): ``!s`` :func:`string ` conversion + This has the same meaning as ``FormattedValue.conversion``. * ``format_spec`` is a :class:`JoinedStr` node representing the formatting of the value, or ``None`` if no format was specified. Both ``conversion`` and ``format_spec`` can be set at the same time. - - .. versionadded:: 3.14 + This has the same meaning as ``FormattedValue.format_spec``. .. class:: List(elts, ctx) @@ -1104,7 +1136,8 @@ Imports names=[ alias(name='x'), alias(name='y'), - alias(name='z')])]) + alias(name='z')], + is_lazy=0)]) .. class:: ImportFrom(module, names, level) @@ -1125,7 +1158,8 @@ Imports alias(name='x'), alias(name='y'), alias(name='z')], - level=0)]) + level=0, + is_lazy=0)]) .. class:: alias(name, asname) @@ -1143,7 +1177,8 @@ Imports names=[ alias(name='a', asname='b'), alias(name='c')], - level=2)]) + level=2, + is_lazy=0)]) Control flow ^^^^^^^^^^^^ @@ -2190,16 +2225,16 @@ Async and await occurrences of the same value (for example, :class:`ast.Add`). -:mod:`ast` helpers ------------------- +:mod:`!ast` helpers +------------------- -Apart from the node classes, the :mod:`ast` module defines these utility functions +Apart from the node classes, the :mod:`!ast` module defines these utility functions and classes for traversing abstract syntax trees: -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1) +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1, module=None) Parse the source into an AST node. Equivalent to ``compile(source, - filename, mode, flags=FLAGS_VALUE, optimize=optimize)``, + filename, mode, flags=FLAGS_VALUE, optimize=optimize, module=module)``, where ``FLAGS_VALUE`` is ``ast.PyCF_ONLY_AST`` if ``optimize <= 0`` and ``ast.PyCF_OPTIMIZED_AST`` otherwise. @@ -2252,6 +2287,9 @@ and classes for traversing abstract syntax trees: The minimum supported version for ``feature_version`` is now ``(3, 7)``. The ``optimize`` argument was added. + .. versionadded:: 3.15 + Added the *module* parameter. + .. function:: unparse(ast_obj) @@ -2407,12 +2445,12 @@ and classes for traversing abstract syntax trees: during traversal. For this a special visitor exists (:class:`NodeTransformer`) that allows modifications. - .. deprecated:: 3.8 + .. deprecated-removed:: 3.8 3.14 Methods :meth:`!visit_Num`, :meth:`!visit_Str`, :meth:`!visit_Bytes`, - :meth:`!visit_NameConstant` and :meth:`!visit_Ellipsis` are deprecated - now and will not be called in future Python versions. Add the - :meth:`visit_Constant` method to handle all constant nodes. + :meth:`!visit_NameConstant` and :meth:`!visit_Ellipsis` will not be called + in Python 3.14+. Add the :meth:`visit_Constant` method instead to handle + all constant nodes. .. class:: NodeTransformer() @@ -2459,7 +2497,7 @@ and classes for traversing abstract syntax trees: node = YourTransformer().visit(node) -.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None, show_empty=False) +.. function:: dump(node, annotate_fields=True, include_attributes=False, *, color=False, indent=None, show_empty=False) Return a formatted dump of the tree in *node*. This is mainly useful for debugging purposes. If *annotate_fields* is true (by default), @@ -2469,6 +2507,10 @@ and classes for traversing abstract syntax trees: numbers and column offsets are not dumped by default. If this is wanted, *include_attributes* can be set to true. + If *color* is ``True``, the returned string is syntax highlighted using + ANSI escape sequences. + If ``False`` (the default), colored output is always disabled. + If *indent* is a non-negative integer or string, then the tree will be pretty-printed with that indent level. An indent level of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) @@ -2503,9 +2545,26 @@ and classes for traversing abstract syntax trees: .. versionchanged:: 3.13 Added the *show_empty* option. - .. versionchanged:: next + .. versionchanged:: 3.15 Omit optional ``Load()`` values by default. + .. versionchanged:: 3.15 + Added the *color* parameter. + + +.. function:: compare(a, b, /, *, compare_attributes=False) + + Recursively compares two ASTs. + + *compare_attributes* affects whether AST attributes are considered + in the comparison. If *compare_attributes* is ``False`` (default), then + attributes are ignored. Otherwise they must all be equal. This + option is useful to check whether the ASTs are structurally equal but + differ in whitespace or similar details. Attributes include line numbers + and column offsets. + + .. versionadded:: 3.14 + .. _ast-compiler-flags: @@ -2542,20 +2601,6 @@ effects on the compilation of a program: .. versionadded:: 3.8 -.. function:: compare(a, b, /, *, compare_attributes=False) - - Recursively compares two ASTs. - - *compare_attributes* affects whether AST attributes are considered - in the comparison. If *compare_attributes* is ``False`` (default), then - attributes are ignored. Otherwise they must all be equal. This - option is useful to check whether the ASTs are structurally equal but - differ in whitespace or similar details. Attributes include line numbers - and column offsets. - - .. versionadded:: 3.14 - - .. _ast-cli: Command-line usage @@ -2563,7 +2608,11 @@ Command-line usage .. versionadded:: 3.9 -The :mod:`ast` module can be executed as a script from the command line. +.. versionchanged:: 3.15 + The output is now syntax highlighted by default. This can be + :ref:`controlled using environment variables `. + +The :mod:`!ast` module can be executed as a script from the command line. It is as simple as: .. code-block:: sh diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index 7831b613bd4a605..713b40d746680af 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -248,3 +248,225 @@ Output in debug mode:: File "../t.py", line 4, in bug raise Exception("not consumed") Exception: not consumed + + +Asynchronous generators best practices +====================================== + +Writing correct and efficient asyncio code requires awareness of certain pitfalls. +This section outlines essential best practices that can save you hours of debugging. + + +Close asynchronous generators explicitly +---------------------------------------- + +It is recommended to manually close the +:term:`asynchronous generator `. If a generator +exits early - for example, due to an exception raised in the body of +an ``async for`` loop - its asynchronous cleanup code may run in an +unexpected context. This can occur after the tasks it depends on have completed, +or during the event loop shutdown when the async-generator's garbage collection +hook is called. + +To avoid this, explicitly close the generator by calling its +:meth:`~agen.aclose` method, or use the :func:`contextlib.aclosing` +context manager:: + + import asyncio + import contextlib + + async def gen(): + yield 1 + yield 2 + + async def func(): + async with contextlib.aclosing(gen()) as g: + async for x in g: + break # Don't iterate until the end + + asyncio.run(func()) + +As noted above, the cleanup code for these asynchronous generators is deferred. +The following example demonstrates that the finalization of an asynchronous +generator can occur in an unexpected order:: + + import asyncio + work_done = False + + async def cursor(): + try: + yield 1 + finally: + assert work_done + + async def rows(): + global work_done + try: + yield 2 + finally: + await asyncio.sleep(0.1) # imitate some async work + work_done = True + + + async def main(): + async for c in cursor(): + async for r in rows(): + break + break + + asyncio.run(main()) + +For this example, we get the following output:: + + unhandled exception during asyncio.run() shutdown + task: ()> exception=AssertionError()> + Traceback (most recent call last): + File "example.py", line 6, in cursor + yield 1 + asyncio.exceptions.CancelledError + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "example.py", line 8, in cursor + assert work_done + ^^^^^^^^^ + AssertionError + +The ``cursor()`` asynchronous generator was finalized before the ``rows`` +generator - an unexpected behavior. + +The example can be fixed by explicitly closing the +``cursor`` and ``rows`` async-generators:: + + async def main(): + async with contextlib.aclosing(cursor()) as cursor_gen: + async for c in cursor_gen: + async with contextlib.aclosing(rows()) as rows_gen: + async for r in rows_gen: + break + break + + +Create asynchronous generators only when the event loop is running +------------------------------------------------------------------ + +It is recommended to create +:term:`asynchronous generators ` only after +the event loop has been created. + +To ensure that asynchronous generators close reliably, the event loop uses the +:func:`sys.set_asyncgen_hooks` function to register callback functions. These +callbacks update the list of running asynchronous generators to keep it in a +consistent state. + +When the :meth:`loop.shutdown_asyncgens() ` +function is called, the running generators are stopped gracefully and the +list is cleared. + +The asynchronous generator invokes the corresponding system hook during its +first iteration. At the same time, the generator records that the hook has +been called and does not call it again. + +Therefore, if iteration begins before the event loop is created, +the event loop will not be able to add the generator to its list of active +generators because the hooks are set after the generator attempts to call them. +Consequently, the event loop will not be able to terminate the generator +if necessary. + +Consider the following example:: + + import asyncio + + async def agenfn(): + try: + yield 10 + finally: + await asyncio.sleep(0) + + + with asyncio.Runner() as runner: + agen = agenfn() + print(runner.run(anext(agen))) + del agen + +Output:: + + 10 + Exception ignored while closing generator : + Traceback (most recent call last): + File "example.py", line 13, in + del agen + ^^^^ + RuntimeError: async generator ignored GeneratorExit + +This example can be fixed as follows:: + + import asyncio + + async def agenfn(): + try: + yield 10 + finally: + await asyncio.sleep(0) + + async def main(): + agen = agenfn() + print(await anext(agen)) + del agen + + asyncio.run(main()) + + +Avoid concurrent iteration and closure of the same generator +------------------------------------------------------------ + +Async generators may be reentered while another +:meth:`~agen.__anext__` / :meth:`~agen.athrow` / :meth:`~agen.aclose` call is in +progress. This may lead to an inconsistent state of the async generator and can +cause errors. + +Let's consider the following example:: + + import asyncio + + async def consumer(): + for idx in range(100): + await asyncio.sleep(0) + message = yield idx + print('received', message) + + async def amain(): + agenerator = consumer() + await agenerator.asend(None) + + fa = asyncio.create_task(agenerator.asend('A')) + fb = asyncio.create_task(agenerator.asend('B')) + await fa + await fb + + asyncio.run(amain()) + +Output:: + + received A + Traceback (most recent call last): + File "test.py", line 38, in + asyncio.run(amain()) + ~~~~~~~~~~~^^^^^^^^^ + File "Lib/asyncio/runners.py", line 204, in run + return runner.run(main) + ~~~~~~~~~~^^^^^^ + File "Lib/asyncio/runners.py", line 127, in run + return self._loop.run_until_complete(task) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ + File "Lib/asyncio/base_events.py", line 719, in run_until_complete + return future.result() + ~~~~~~~~~~~~~^^ + File "test.py", line 36, in amain + await fb + RuntimeError: anext(): asynchronous generator is already running + + +Therefore, it is recommended to avoid using asynchronous generators in parallel +tasks or across multiple event loops. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 91970c282391f7a..79c9516cda2d60b 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -4,7 +4,7 @@ .. _asyncio-event-loop: ========== -Event Loop +Event loop ========== **Source code:** :source:`Lib/asyncio/events.py`, @@ -105,7 +105,7 @@ This documentation page contains the following sections: .. _asyncio-event-loop-methods: -Event Loop Methods +Event loop methods ================== Event loops have **low-level** APIs for the following: @@ -297,13 +297,20 @@ clocks to track time. are called is undefined. The optional positional *args* will be passed to the callback when - it is called. If you want the callback to be called with keyword - arguments use :func:`functools.partial`. + it is called. Use :func:`functools.partial` + :ref:`to pass keyword arguments ` to + *callback*. An optional keyword-only *context* argument allows specifying a custom :class:`contextvars.Context` for the *callback* to run in. The current context is used when no *context* is provided. + .. note:: + + For performance, callbacks scheduled with :meth:`loop.call_later` + may run up to one clock-resolution early (see + ``time.get_clock_info('monotonic').resolution``). + .. versionchanged:: 3.7 The *context* keyword-only parameter was added. See :pep:`567` for more details. @@ -324,6 +331,12 @@ clocks to track time. An instance of :class:`asyncio.TimerHandle` is returned which can be used to cancel the callback. + .. note:: + + For performance, callbacks scheduled with :meth:`loop.call_at` + may run up to one clock-resolution early (see + ``time.get_clock_info('monotonic').resolution``). + .. versionchanged:: 3.7 The *context* keyword-only parameter was added. See :pep:`567` for more details. @@ -348,7 +361,7 @@ clocks to track time. The :func:`asyncio.sleep` function. -Creating Futures and Tasks +Creating futures and tasks ^^^^^^^^^^^^^^^^^^^^^^^^^^ .. method:: loop.create_future() @@ -611,6 +624,12 @@ Opening network connections to bind the socket locally. The *local_host* and *local_port* are looked up using :meth:`getaddrinfo`. + .. note:: + + On Windows, when using the proactor event loop with ``local_addr=None``, + an :exc:`OSError` with :attr:`!errno.WSAEINVAL` will be raised + when running it. + * *remote_addr*, if given, is a ``(remote_host, remote_port)`` tuple used to connect the socket to a remote address. The *remote_host* and *remote_port* are looked up using :meth:`getaddrinfo`. @@ -943,7 +962,7 @@ Transferring files .. versionadded:: 3.7 -TLS Upgrade +TLS upgrade ^^^^^^^^^^^ .. method:: loop.start_tls(transport, protocol, \ @@ -1016,8 +1035,8 @@ Watching file descriptors .. method:: loop.add_writer(fd, callback, *args) Start monitoring the *fd* file descriptor for write availability and - invoke *callback* with the specified arguments once *fd* is available for - writing. + invoke *callback* with the specified arguments *args* once *fd* is + available for writing. Any preexisting callback registered for *fd* is cancelled and replaced by *callback*. @@ -1290,7 +1309,8 @@ Unix signals .. method:: loop.add_signal_handler(signum, callback, *args) - Set *callback* as the handler for the *signum* signal. + Set *callback* as the handler for the *signum* signal, + passing *args* as positional arguments. The callback will be invoked by *loop*, along with other queued callbacks and runnable coroutines of that event loop. Unlike signal handlers @@ -1325,7 +1345,8 @@ Executing code in thread or process pools .. awaitablemethod:: loop.run_in_executor(executor, func, *args) - Arrange for *func* to be called in the specified executor. + Arrange for *func* to be called in the specified executor + passing *args* as positional arguments. The *executor* argument should be an :class:`concurrent.futures.Executor` instance. The default executor is used if *executor* is ``None``. @@ -1410,7 +1431,7 @@ Executing code in thread or process pools :class:`~concurrent.futures.ThreadPoolExecutor`. -Error Handling API +Error handling API ^^^^^^^^^^^^^^^^^^ Allows customizing how exceptions are handled in the event loop. @@ -1513,7 +1534,7 @@ Enabling debug mode The :ref:`debug mode of asyncio `. -Running Subprocesses +Running subprocesses ^^^^^^^^^^^^^^^^^^^^ Methods described in this subsections are low-level. In regular @@ -1613,6 +1634,9 @@ async/await code consider using the high-level conforms to the :class:`asyncio.SubprocessTransport` base class and *protocol* is an object instantiated by the *protocol_factory*. + If the transport is closed or is garbage collected, the child process + is killed if it is still running. + .. method:: loop.subprocess_shell(protocol_factory, cmd, *, \ stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ stderr=subprocess.PIPE, **kwargs) @@ -1636,6 +1660,9 @@ async/await code consider using the high-level conforms to the :class:`SubprocessTransport` base class and *protocol* is an object instantiated by the *protocol_factory*. + If the transport is closed or is garbage collected, the child process + is killed if it is still running. + .. note:: It is the application's responsibility to ensure that all whitespace and special characters are quoted appropriately to avoid `shell injection @@ -1645,7 +1672,7 @@ async/await code consider using the high-level are going to be used to construct shell commands. -Callback Handles +Callback handles ================ .. class:: Handle @@ -1688,7 +1715,7 @@ Callback Handles .. versionadded:: 3.7 -Server Objects +Server objects ============== Server objects are created by :meth:`loop.create_server`, @@ -1831,7 +1858,7 @@ Do not instantiate the :class:`Server` class directly. .. _asyncio-event-loops: .. _asyncio-event-loop-implementations: -Event Loop Implementations +Event loop implementations ========================== asyncio ships with two different event loop implementations: @@ -1944,10 +1971,10 @@ callback uses the :meth:`loop.call_later` method to reschedule itself after 5 seconds, and then stops the event loop:: import asyncio - import datetime + import datetime as dt def display_date(end_time, loop): - print(datetime.datetime.now()) + print(dt.datetime.now()) if (loop.time() + 1.0) < end_time: loop.call_later(1, display_date, end_time, loop) else: @@ -2028,7 +2055,7 @@ Wait until a file descriptor received some data using the Set signal handlers for SIGINT and SIGTERM ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -(This ``signals`` example only works on Unix.) +(This ``signal`` example only works on Unix.) Register handlers for signals :const:`~signal.SIGINT` and :const:`~signal.SIGTERM` using the :meth:`loop.add_signal_handler` method:: diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 32771ba72e00026..195d99123dbd367 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -75,6 +75,7 @@ Future Functions Deprecation warning is emitted if *future* is not a Future-like object and *loop* is not specified and there is no running event loop. +.. _asyncio-future-obj: Future Object ============= @@ -100,6 +101,8 @@ Future Object implementations can inject their own optimized implementations of a Future object. + Futures are :ref:`generic ` over the type of their results. + .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. @@ -195,6 +198,10 @@ Future Object Otherwise, change the Future's state to *cancelled*, schedule the callbacks, and return ``True``. + The optional string argument *msg* is passed as the argument to the + :exc:`CancelledError` exception raised when a cancelled Future + is awaited. + .. versionchanged:: 3.9 Added the *msg* parameter. diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 7c08d65f26bc27e..58f77feb3119841 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -390,11 +390,11 @@ Subprocess Transports Return the transport for the communication pipe corresponding to the integer file descriptor *fd*: - * ``0``: readable streaming transport of the standard input (*stdin*), + * ``0``: writable streaming transport of the standard input (*stdin*), or :const:`None` if the subprocess was not created with ``stdin=PIPE`` - * ``1``: writable streaming transport of the standard output (*stdout*), + * ``1``: readable streaming transport of the standard output (*stdout*), or :const:`None` if the subprocess was not created with ``stdout=PIPE`` - * ``2``: writable streaming transport of the standard error (*stderr*), + * ``2``: readable streaming transport of the standard error (*stderr*), or :const:`None` if the subprocess was not created with ``stderr=PIPE`` * other *fd*: :const:`None` @@ -1037,7 +1037,7 @@ The subprocess is created by the :meth:`loop.subprocess_exec` method:: # low-level APIs. loop = asyncio.get_running_loop() - code = 'import datetime; print(datetime.datetime.now())' + code = 'import datetime as dt; print(dt.datetime.now())' exit_future = asyncio.Future(loop=loop) # Create the subprocess controlled by DateProtocol; diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index 963bc1fb82c12f1..a9735ae80652df6 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -107,7 +107,7 @@ Queue The queue can no longer grow. Future calls to :meth:`~Queue.put` raise :exc:`QueueShutDown`. Currently blocked callers of :meth:`~Queue.put` will be unblocked - and will raise :exc:`QueueShutDown` in the formerly blocked thread. + and will raise :exc:`QueueShutDown` in the formerly awaiting task. If *immediate* is false (the default), the queue can be wound down normally with :meth:`~Queue.get` calls to extract tasks @@ -120,9 +120,10 @@ Queue raise :exc:`QueueShutDown`. If *immediate* is true, the queue is terminated immediately. - The queue is drained to be completely empty. All callers of - :meth:`~Queue.join` are unblocked regardless of the number - of unfinished tasks. Blocked callers of :meth:`~Queue.get` + The queue is drained to be completely empty and the count + of unfinished tasks is reduced by the number of tasks drained. + If unfinished tasks is zero, callers of :meth:`~Queue.join` + are unblocked. Also, blocked callers of :meth:`~Queue.get` are unblocked and will raise :exc:`QueueShutDown` because the queue is empty. diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 90c90862ca1ed3a..05445219510ca54 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -316,11 +316,15 @@ StreamWriter If that fails, the data is queued in an internal write buffer until it can be sent. + The *data* buffer should be a bytes, bytearray, or C-contiguous one-dimensional + memoryview object. + The method should be used along with the ``drain()`` method:: stream.write(data) await stream.drain() + .. method:: writelines(data) The method writes a list (or any iterable) of bytes to the underlying socket diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 03e76bc868905ee..a6514649bf9a0a8 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -76,6 +76,9 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_exec` for other parameters. + If the process object is garbage collected while the process is still + running, the child process will be killed. + .. versionchanged:: 3.10 Removed the *loop* parameter. @@ -95,6 +98,9 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_shell` for other parameters. + If the process object is garbage collected while the process is still + running, the child process will be killed. + .. important:: It is the application's responsibility to ensure that all whitespace and @@ -305,8 +311,16 @@ their completion. A ``None`` value indicates that the process has not terminated yet. - A negative value ``-N`` indicates that the child was terminated - by signal ``N`` (POSIX only). + For processes created with :func:`~asyncio.create_subprocess_exec`, a negative + value ``-N`` indicates that the child was terminated by signal ``N`` + (POSIX only). + + For processes created with :func:`~asyncio.create_subprocess_shell`, the + return code reflects the exit status of the shell itself (e.g. ``/bin/sh``), + which may map signals to codes such as ``128+N``. See the + documentation of the shell (for example, the Bash manual's Exit Status) + for details. + .. _asyncio-subprocess-threads: @@ -345,7 +359,7 @@ function:: import sys async def get_date(): - code = 'import datetime; print(datetime.datetime.now())' + code = 'import datetime as dt; print(dt.datetime.now())' # Create the subprocess; redirect the standard output # into a pipe. diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 968c812ee3c8e6f..f9e98e05cab7aca 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -157,7 +157,7 @@ Event Clear (unset) the event. - Tasks awaiting on :meth:`~Event.wait` will now block until the + Subsequent tasks awaiting on :meth:`~Event.wait` will now block until the :meth:`~Event.set` method is called again. .. method:: is_set() diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index b19ffa8213a9713..64f0810777e41b9 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -2,7 +2,7 @@ ==================== -Coroutines and Tasks +Coroutines and tasks ==================== This section outlines high-level asyncio APIs to work with coroutines @@ -231,7 +231,7 @@ A good example of a low-level function that returns a Future object is :meth:`loop.run_in_executor`. -Creating Tasks +Creating tasks ============== **Source code:** :source:`Lib/asyncio/tasks.py` @@ -300,7 +300,7 @@ Creating Tasks Added the *eager_start* parameter by passing on all *kwargs*. -Task Cancellation +Task cancellation ================= Tasks can easily and safely be cancelled. @@ -324,7 +324,7 @@ remove the cancellation state. .. _taskgroups: -Task Groups +Task groups =========== Task groups combine a task creation API with a convenient @@ -355,6 +355,34 @@ and reliable way to wait for all tasks in the group to finish. Passes on all *kwargs* to :meth:`loop.create_task` + .. method:: cancel() + + Cancel the task group. This is a non-exceptional, early exit of the + task group's lifetime -- useful once the group's goal has been met or + its services no longer needed. + + :meth:`~asyncio.Task.cancel` will be called on any tasks in the group that + aren't yet done, as well as the parent (body) of the group. The task group + context manager will exit *without* :exc:`asyncio.CancelledError` being raised. + + If :meth:`cancel` is called before entering the task group, the group will be + cancelled upon entry. This is useful for patterns where one piece of + code passes an unused :class:`asyncio.TaskGroup` instance to another in order to have + the ability to cancel anything run within the group. + + :meth:`cancel` is idempotent and may be called after the task group has + already exited. + + Some ways to use :meth:`cancel`: + + * call it from the task group body based on some condition or event + * pass the task group instance to child tasks via :meth:`create_task`, allowing a child + task to conditionally cancel the entire entire group + * pass the task group instance or bound :meth:`cancel` method to some other task *before* + opening the task group, allowing remote cancellation + + .. versionadded:: 3.15 + Example:: async def main(): @@ -366,7 +394,8 @@ Example:: The ``async with`` statement will wait for all tasks in the group to finish. While waiting, new tasks may still be added to the group (for example, by passing ``tg`` into one of the coroutines -and calling ``tg.create_task()`` in that coroutine). +and calling ``tg.create_task()`` in that coroutine). There is also opportunity to +request termination of the entire task group with ``tg.cancel()``, based on some condition. Once the last task has finished and the ``async with`` block is exited, no new tasks may be added to the group. @@ -427,53 +456,6 @@ reported by :meth:`asyncio.Task.cancelling`. Improved handling of simultaneous internal and external cancellations and correct preservation of cancellation counts. -Terminating a Task Group ------------------------- - -While terminating a task group is not natively supported by the standard -library, termination can be achieved by adding an exception-raising task -to the task group and ignoring the raised exception: - -.. code-block:: python - - import asyncio - from asyncio import TaskGroup - - class TerminateTaskGroup(Exception): - """Exception raised to terminate a task group.""" - - async def force_terminate_task_group(): - """Used to force termination of a task group.""" - raise TerminateTaskGroup() - - async def job(task_id, sleep_time): - print(f'Task {task_id}: start') - await asyncio.sleep(sleep_time) - print(f'Task {task_id}: done') - - async def main(): - try: - async with TaskGroup() as group: - # spawn some tasks - group.create_task(job(1, 0.5)) - group.create_task(job(2, 1.5)) - # sleep for 1 second - await asyncio.sleep(1) - # add an exception-raising task to force the group to terminate - group.create_task(force_terminate_task_group()) - except* TerminateTaskGroup: - pass - - asyncio.run(main()) - -Expected output: - -.. code-block:: text - - Task 1: start - Task 2: start - Task 1: done - Sleeping ======== @@ -498,13 +480,13 @@ Sleeping for 5 seconds:: import asyncio - import datetime + import datetime as dt async def display_date(): loop = asyncio.get_running_loop() end_time = loop.time() + 5.0 while True: - print(datetime.datetime.now()) + print(dt.datetime.now()) if (loop.time() + 1.0) >= end_time: break await asyncio.sleep(1) @@ -519,7 +501,7 @@ Sleeping Raises :exc:`ValueError` if *delay* is :data:`~math.nan`. -Running Tasks Concurrently +Running tasks concurrently ========================== .. awaitablefunction:: gather(*aws, return_exceptions=False) @@ -557,7 +539,7 @@ Running Tasks Concurrently provides stronger safety guarantees than *gather* for scheduling a nesting of subtasks: if a task (or a subtask, a task scheduled by a task) raises an exception, *TaskGroup* will, while *gather* will not, - cancel the remaining scheduled tasks). + cancel the remaining scheduled tasks. .. _asyncio_example_gather: @@ -621,7 +603,7 @@ Running Tasks Concurrently .. _eager-task-factory: -Eager Task Factory +Eager task factory ================== .. function:: eager_task_factory(loop, coro, *, name=None, context=None) @@ -664,7 +646,7 @@ Eager Task Factory .. versionadded:: 3.12 -Shielding From Cancellation +Shielding from cancellation =========================== .. awaitablefunction:: shield(aw) @@ -771,6 +753,9 @@ Timeouts An :ref:`asynchronous context manager ` for cancelling overdue coroutines. + Prefer using :func:`asyncio.timeout` or :func:`asyncio.timeout_at` + rather than instantiating :class:`!Timeout` directly. + ``when`` should be an absolute time at which the context should time out, as measured by the event loop's clock: @@ -891,7 +876,7 @@ Timeouts Raises :exc:`TimeoutError` instead of :exc:`asyncio.TimeoutError`. -Waiting Primitives +Waiting primitives ================== .. function:: wait(aws, *, timeout=None, return_when=ALL_COMPLETED) @@ -1011,7 +996,7 @@ Waiting Primitives or as a plain :term:`iterator` (previously it was only a plain iterator). -Running in Threads +Running in threads ================== .. function:: to_thread(func, /, *args, **kwargs) @@ -1071,7 +1056,7 @@ Running in Threads .. versionadded:: 3.9 -Scheduling From Other Threads +Scheduling from other threads ============================= .. function:: run_coroutine_threadsafe(coro, loop) @@ -1193,8 +1178,9 @@ Introspection .. versionadded:: 3.4 +.. _asyncio-task-obj: -Task Object +Task object =========== .. class:: Task(coro, *, loop=None, name=None, context=None, eager_start=False) @@ -1220,8 +1206,8 @@ Task Object To cancel a running Task use the :meth:`cancel` method. Calling it will cause the Task to throw a :exc:`CancelledError` exception into - the wrapped coroutine. If a coroutine is awaiting on a Future - object during cancellation, the Future object will be cancelled. + the wrapped coroutine. If a coroutine is awaiting on a future-like + object during cancellation, the awaited object will be cancelled. :meth:`cancelled` can be used to check if the Task was cancelled. The method returns ``True`` if the wrapped coroutine did not @@ -1244,6 +1230,9 @@ Task Object blocks. If the coroutine returns or raises without blocking, the task will be finished eagerly and will skip scheduling to the event loop. + Tasks are :ref:`generic ` over the return type of their wrapped + coroutines. + .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. @@ -1410,6 +1399,10 @@ Task Object the cancellation, it needs to call :meth:`Task.uncancel` in addition to catching the exception. + If the Task being cancelled is currently awaiting on a future-like + object, that awaited object will also be cancelled. This cancellation + propagates down the entire chain of awaited objects. + .. versionchanged:: 3.9 Added the *msg* parameter. diff --git a/Doc/library/asyncio-threading.rst b/Doc/library/asyncio-threading.rst new file mode 100644 index 000000000000000..526901a2e7eb206 --- /dev/null +++ b/Doc/library/asyncio-threading.rst @@ -0,0 +1,154 @@ +.. currentmodule:: asyncio + +.. _asyncio-threading: + +asyncio and free-threaded Python +================================ + +asyncio uses an event loop as a scheduler to enable highly efficient +concurrency by switching between tasks to allow non-blocking I/O +operations. This results in better performance for I/O-bound use +cases. It also allows off-loading CPU-bound work to a thread or +process pool, but that is still limited by the :term:`global +interpreter lock` in CPython. + +However, in :ref:`free-threaded Python `, +the GIL is disabled and Python can run true multi-threaded code. This +means that asyncio can now take advantage of multiple CPU cores without +the limitations imposed by the GIL. + +Since Python 3.14, asyncio has first-class support for free-threaded +Python, and the implementation of asyncio is safe to use in a +multi-threaded environment. + +A single event loop on one core can handle many connections +concurrently, but the Python code that runs to handle each one still +executes serially. Once requests involve a non-trivial amount of +per-request computation, that handling becomes the bottleneck, and a +single core can no longer keep up. Combining asyncio with threads is +most useful here: by running an event loop per thread, the handling of +different requests can run in parallel across multiple CPU cores. It is +also useful when you need to run blocking or CPU-bound code from an +asyncio application. + + +.. seealso:: + + `Scaling asyncio on Free-Threaded Python + `__, + a blog post by Kumar Aditya which explains the internal changes + that make asyncio safe and efficient under free-threaded Python, + together with benchmarks of the resulting improvements. + + +Thread safety considerations +---------------------------- + +While asyncio is designed to be thread-safe in a free-threaded Python +environment, there are still some considerations to keep in mind when +using asyncio with threads: + +1. **Event loop**: Each thread should have its own event loop which + should not be shared across threads. This ensures that the event loop + can manage its own tasks and callbacks without interference from + other threads. + +2. **Task management**: Tasks and futures created in one thread should + not be awaited or manipulated from another thread. + +3. **Thread-safe APIs**: When interacting with asyncio from multiple + threads, it's important to use thread-safe APIs provided by asyncio, + such as :func:`asyncio.run_coroutine_threadsafe` for submitting + coroutines to an event loop from another thread. If you need to + call a callback from a different thread, you can use + :meth:`loop.call_soon_threadsafe` to schedule it safely. + +4. **Synchronization**: The synchronization primitives provided by + asyncio (like :class:`asyncio.Lock` and :class:`asyncio.Event`) + are not designed to be used across threads. If you need to + synchronize between threads, you should use the synchronization + primitives from the :mod:`threading` module instead. + + +Using asyncio with threads +-------------------------- + +asyncio supports running one event loop per thread, which allows you to +take advantage of multiple CPU cores in a free-threaded Python +environment. Each thread can run its own event loop, and tasks can be +scheduled on those loops independently. + +Here's an example of how to use asyncio with threads:: + + import asyncio + import threading + + async def worker(name: str) -> None: + print(f"Worker {name} starting") + await asyncio.sleep(1) + print(f"Worker {name} done") + + def run_loop(name: str) -> None: + asyncio.run(worker(name)) + + threads = [ + threading.Thread(target=run_loop, args=(f"T{i}",)) + for i in range(4) + ] + for t in threads: + t.start() + for t in threads: + t.join() + +In this example, each thread creates its own event loop with +:func:`asyncio.run` and runs a coroutine on it. The threads execute +concurrently, and in a free-threaded build they can run on separate +CPU cores in parallel. + + +Producer/consumer across threads +-------------------------------- + +When a regular (non-asyncio) thread needs to hand work to an asyncio +event loop running in another thread, use a thread-safe primitive such +as :class:`queue.Queue` rather than :class:`asyncio.Queue`, which is +only safe within a single event loop.:: + + import asyncio + import queue + import threading + + def producer(q: queue.Queue[int]) -> None: + for i in range(5): + print(f"Producing {i}") + q.put(i) + q.shutdown() + + async def consumer(q: queue.Queue[int]) -> None: + while True: + try: + item = q.get_nowait() + except queue.Empty: + await asyncio.sleep(0.1) + continue + except queue.ShutDown: + break + print(f"Consumed {item}") + await asyncio.sleep(item) + + q: queue.Queue[int] = queue.Queue() + consumer_thread = threading.Thread( + target=lambda: asyncio.run(consumer(q)) + ) + consumer_thread.start() + producer(q) + consumer_thread.join() + +The producer runs on the main thread while the consumer runs inside an +event loop on its own thread, yet they communicate safely through +``queue.Queue``. When the queue is empty the consumer sleeps briefly +and tries again. When the producer is done it calls +:meth:`~queue.Queue.shutdown`, which causes subsequent +:meth:`~queue.Queue.get_nowait` calls to raise :exc:`queue.ShutDown` +so the consumer can exit cleanly. + diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 7d368dae49dc1d7..90a465f3e1d3af4 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -29,6 +29,11 @@ database connection libraries, distributed task queues, etc. asyncio is often a perfect fit for IO-bound and high-level **structured** network code. +.. seealso:: + + :ref:`a-conceptual-overview-of-asyncio` + Explanation of the fundamentals of asyncio. + asyncio provides a set of **high-level** APIs to: * :ref:`run Python coroutines ` concurrently and @@ -74,6 +79,10 @@ You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`: >>> await asyncio.sleep(10, result='hello') 'hello' +This REPL provides limited compatibility with :envvar:`PYTHON_BASIC_REPL`. +It is recommended that the default REPL is used +for full functionality and the latest features. + .. audit-event:: cpython.run_stdin "" "" .. versionchanged:: 3.12.5 (also 3.11.10, 3.10.15, 3.9.20, and 3.8.20) @@ -119,6 +128,7 @@ You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`: asyncio-api-index.rst asyncio-llapi-index.rst asyncio-dev.rst + asyncio-threading.rst .. note:: The source code for asyncio can be found in :source:`Lib/asyncio/`. diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index 02d2f0807df8f69..b5caf5502d0e1c6 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -4,14 +4,11 @@ .. module:: atexit :synopsis: Register and execute cleanup functions. -.. moduleauthor:: Skip Montanaro -.. sectionauthor:: Skip Montanaro - -------------- -The :mod:`atexit` module defines functions to register and unregister cleanup +The :mod:`!atexit` module defines functions to register and unregister cleanup functions. Functions thus registered are automatically executed upon normal -interpreter termination. :mod:`atexit` runs these functions in the *reverse* +interpreter termination. :mod:`!atexit` runs these functions in the *reverse* order in which they were registered; if you register ``A``, ``B``, and ``C``, at interpreter termination time they will be run in the order ``C``, ``B``, ``A``. @@ -64,7 +61,7 @@ a cleanup function is undefined. Remove *func* from the list of functions to be run at interpreter shutdown. :func:`unregister` silently does nothing if *func* was not previously registered. If *func* has been registered more than once, every occurrence - of that function in the :mod:`atexit` call stack will be removed. Equality + of that function in the :mod:`!atexit` call stack will be removed. Equality comparisons (``==``) are used internally during unregistration, so function references do not need to have matching identities. @@ -72,14 +69,14 @@ a cleanup function is undefined. .. seealso:: Module :mod:`readline` - Useful example of :mod:`atexit` to read and write :mod:`readline` history + Useful example of :mod:`!atexit` to read and write :mod:`readline` history files. .. _atexit-example: -:mod:`atexit` Example ---------------------- +:mod:`!atexit` Example +---------------------- The following simple example demonstrates how a module can initialize a counter from a file when it is imported and save the counter's updated value diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 529a72424438203..8af40a2f8a65e3f 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -16,8 +16,10 @@ This module provides functions for encoding binary data to printable ASCII characters and decoding such encodings back to binary data. This includes the :ref:`encodings specified in ` -:rfc:`4648` (Base64, Base32 and Base16) -and the non-standard :ref:`Base85 encodings `. +:rfc:`4648` (Base64, Base32 and Base16), the :ref:`Base85 encoding +` specified in `PDF 2.0 +`_, and non-standard variants +of Base85 used elsewhere. There are two interfaces provided by this module. The modern interface supports encoding :term:`bytes-like objects ` to ASCII @@ -51,7 +53,7 @@ The :rfc:`4648` encodings are suitable for encoding binary data so that it can b safely sent by email, used as parts of URLs, or included as part of an HTTP POST request. -.. function:: b64encode(s, altchars=None) +.. function:: b64encode(s, altchars=None, *, padded=True, wrapcol=0) Encode the :term:`bytes-like object` *s* using Base64 and return the encoded :class:`bytes`. @@ -61,11 +63,20 @@ POST request. This allows an application to e.g. generate URL or filesystem safe Base64 strings. The default is ``None``, for which the standard Base64 alphabet is used. - May assert or raise a :exc:`ValueError` if the length of *altchars* is not 2. Raises a - :exc:`TypeError` if *altchars* is not a :term:`bytes-like object`. + If *padded* is true (default), pad the encoded data with the '=' + character to a size multiple of 4. + If *padded* is false, do not add the pad characters. + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. -.. function:: b64decode(s, altchars=None, validate=False) + .. versionchanged:: 3.15 + Added the *padded* and *wrapcol* parameters. + + +.. function:: b64decode(s, altchars=None, validate=False, *, padded=True, canonical=False) + b64decode(s, altchars=None, validate=True, *, ignorechars, padded=True, canonical=False) Decode the Base64 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. @@ -74,18 +85,47 @@ POST request. of length 2 which specifies the alternative alphabet used instead of the ``+`` and ``/`` characters. + If *padded* is true, the last group of 4 base 64 alphabet characters must + be padded with the '=' character. + If *padded* is false, padding is neither required nor recognized: + the '=' character is not treated as padding but as a non-alphabet + character, which means it is silently discarded when *validate* is false, + or causes an :exc:`~binascii.Error` when *validate* is true unless + b'=' is included in *ignorechars*. + A :exc:`binascii.Error` exception is raised if *s* is incorrectly padded. - If *validate* is ``False`` (the default), characters that are neither - in the normal base-64 alphabet nor the alternative alphabet are - discarded prior to the padding check. If *validate* is ``True``, - these non-alphabet characters in the input result in a - :exc:`binascii.Error`. + If *ignorechars* is specified, it should be a :term:`bytes-like object` + containing characters to ignore from the input when *validate* is true. + If *ignorechars* contains the pad character ``'='``, the pad characters + presented before the end of the encoded data and the excess pad characters + will be ignored. + The default value of *validate* is ``True`` if *ignorechars* is specified, + ``False`` otherwise. + + If *validate* is false, characters that are neither + in the normal base-64 alphabet nor (if *ignorechars* is not specified) + the alternative alphabet are + discarded prior to the padding check, but the ``+`` and ``/`` characters + keep their meaning if they are not in *altchars* (they will be discarded + in future Python versions). + + If *validate* is true, these non-alphabet characters in the input + result in a :exc:`binascii.Error`. + + If *canonical* is true, non-zero padding bits are rejected. + See :func:`binascii.a2b_base64` for details. For more information about the strict base64 check, see :func:`binascii.a2b_base64` - May assert or raise a :exc:`ValueError` if the length of *altchars* is not 2. + .. versionchanged:: 3.15 + Added the *canonical*, *ignorechars*, and *padded* parameters. + + .. deprecated:: 3.15 + Accepting the ``+`` and ``/`` characters with an alternative alphabet + is now deprecated. + .. function:: standard_b64encode(s) @@ -99,16 +139,19 @@ POST request. Base64 alphabet and return the decoded :class:`bytes`. -.. function:: urlsafe_b64encode(s) +.. function:: urlsafe_b64encode(s, *, padded=True) Encode :term:`bytes-like object` *s* using the URL- and filesystem-safe alphabet, which substitutes ``-`` instead of ``+`` and ``_`` instead of ``/`` in the standard Base64 alphabet, and return the encoded :class:`bytes`. The result - can still contain ``=``. + can still contain ``=`` if *padded* is true (default). + + .. versionchanged:: 3.15 + Added the *padded* parameter. -.. function:: urlsafe_b64decode(s) +.. function:: urlsafe_b64decode(s, *, padded=False) Decode :term:`bytes-like object` or ASCII string *s* using the URL- and filesystem-safe @@ -116,14 +159,32 @@ POST request. ``/`` in the standard Base64 alphabet, and return the decoded :class:`bytes`. + .. versionchanged:: 3.15 + Added the *padded* parameter. + Padding of input is no longer required by default. + + .. deprecated:: 3.15 + Accepting the ``+`` and ``/`` characters is now deprecated. + -.. function:: b32encode(s) +.. function:: b32encode(s, *, padded=True, wrapcol=0) Encode the :term:`bytes-like object` *s* using Base32 and return the encoded :class:`bytes`. + If *padded* is true (default), pad the encoded data with the '=' + character to a size multiple of 8. + If *padded* is false, do not add the pad characters. -.. function:: b32decode(s, casefold=False, map01=None) + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not add any newlines. + + .. versionchanged:: 3.15 + Added the *padded* and *wrapcol* parameters. + + +.. function:: b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b'', canonical=False) Decode the Base32 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. @@ -139,20 +200,39 @@ POST request. digit 0 is always mapped to the letter O). For security purposes the default is ``None``, so that 0 and 1 are not allowed in the input. + If *padded* is true, the last group of 8 base 32 alphabet characters must + be padded with the '=' character. + If *padded* is false, padding is neither required nor recognized: + the '=' character is not treated as padding but as a non-alphabet + character, which means it raises an :exc:`~binascii.Error` unless + b'=' is included in *ignorechars*. + + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + + If *canonical* is true, non-zero padding bits are rejected. + See :func:`binascii.a2b_base32` for details. + A :exc:`binascii.Error` is raised if *s* is incorrectly padded or if there are non-alphabet characters present in the input. + .. versionchanged:: 3.15 + Added the *canonical*, *ignorechars*, and *padded* parameters. -.. function:: b32hexencode(s) + +.. function:: b32hexencode(s, *, padded=True, wrapcol=0) Similar to :func:`b32encode` but uses the Extended Hex Alphabet, as defined in :rfc:`4648`. .. versionadded:: 3.10 + .. versionchanged:: 3.15 + Added the *padded* and *wrapcol* parameters. + -.. function:: b32hexdecode(s, casefold=False) +.. function:: b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b'', canonical=False) Similar to :func:`b32decode` but uses the Extended Hex Alphabet, as defined in :rfc:`4648`. @@ -164,14 +244,24 @@ POST request. .. versionadded:: 3.10 + .. versionchanged:: 3.15 + Added the *canonical*, *ignorechars*, and *padded* parameters. + -.. function:: b16encode(s) +.. function:: b16encode(s, *, wrapcol=0) Encode the :term:`bytes-like object` *s* using Base16 and return the encoded :class:`bytes`. + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not add any newlines. + + .. versionchanged:: 3.15 + Added the *wrapcol* parameter. + -.. function:: b16decode(s, casefold=False) +.. function:: b16decode(s, casefold=False, *, ignorechars=b'') Decode the Base16 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. @@ -180,28 +270,44 @@ POST request. lowercase alphabet is acceptable as input. For security purposes, the default is ``False``. + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + A :exc:`binascii.Error` is raised if *s* is incorrectly padded or if there are non-alphabet characters present in the input. + .. versionchanged:: 3.15 + Added the *ignorechars* parameter. + + .. _base64-base-85: Base85 Encodings ----------------- -Base85 encoding is not formally specified but rather a de facto standard, -thus different systems perform the encoding differently. +Base85 encoding is a family of algorithms which represent four bytes +using five ASCII characters. Originally implemented in the Unix +``btoa(1)`` utility, a version of it was later adopted by Adobe in the +PostScript language and is standardized in PDF 2.0 (ISO 32000-2). +This version, in both its ``btoa`` and PDF variants, is implemented by +:func:`a85encode`. -The :func:`a85encode` and :func:`b85encode` functions in this module are two implementations of -the de facto standard. You should call the function with the Base85 -implementation used by the software you intend to work with. +A separate version, using a different output character set, was +defined as an April Fool's joke in :rfc:`1924` but is now used by Git +and other software. This version is implemented by :func:`b85encode`. -The two functions present in this module differ in how they handle the following: +Finally, a third version, using yet another output character set +designed for safe inclusion in programming language strings, is +defined by ZeroMQ and implemented here by :func:`z85encode`. -* Whether to include enclosing ``<~`` and ``~>`` markers -* Whether to include newline characters -* The set of ASCII characters used for encoding -* Handling of null bytes +The functions present in this module differ in how they handle the following: + +* Whether to include and expect enclosing ``<~`` and ``~>`` markers. +* Whether to fold the input into multiple lines. +* The set of ASCII characters used for encoding. +* Compact encodings of sequences of spaces and null bytes. +* The encoding of zero-padding bytes applied to the input. Refer to the documentation of the individual functions for more information. @@ -212,78 +318,137 @@ Refer to the documentation of the individual functions for more information. *foldspaces* is an optional flag that uses the special short sequence 'y' instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Ascii85 encoding. + feature is not supported by the standard encoding used in PDF. - *wrapcol* controls whether the output should have newline (``b'\n'``) - characters added to it. If this is non-zero, each output line will be - at most this many characters long, excluding the trailing newline. + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. - *pad* controls whether the input is padded to a multiple of 4 - before encoding. Note that the ``btoa`` implementation always pads. + *pad* controls whether zero-padding applied to the end of the input + is fully retained in the output encoding, as done by ``btoa``, + producing an exact multiple of 5 bytes of output. This is not part + of the standard encoding used in PDF, as it does not preserve the + length of the data. - *adobe* controls whether the encoded byte sequence is framed with ``<~`` - and ``~>``, which is used by the Adobe implementation. + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. Note + that while ASCII85Decode streams in PDF documents *must* be + terminated with ``~>``, they *must not* use a leading ``<~``. .. versionadded:: 3.4 -.. function:: a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v') +.. function:: a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v', canonical=False) Decode the Ascii85 encoded :term:`bytes-like object` or ASCII string *b* and return the decoded :class:`bytes`. *foldspaces* is a flag that specifies whether the 'y' short sequence should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). - This feature is not supported by the "standard" Ascii85 encoding. + This feature is not supported by the standard Ascii85 encoding used in + PDF and PostScript. - *adobe* controls whether the input sequence is in Adobe Ascii85 format - (i.e. is framed with <~ and ~>). + *adobe* controls whether the ``<~`` and ``~>`` markers are + present. While the leading ``<~`` is not required, the input must + end with ``~>``, or a :exc:`ValueError` is raised. - *ignorechars* should be a :term:`bytes-like object` or ASCII string - containing characters to ignore - from the input. This should only contain whitespace characters, and by + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + This should only contain whitespace characters, and by default contains all whitespace characters in ASCII. + If *canonical* is true, non-canonical encodings are rejected. + See :func:`binascii.a2b_ascii85` for details. + .. versionadded:: 3.4 + .. versionchanged:: 3.15 + Added the *canonical* parameter. + Single-character final groups are now always rejected as encoding + violations. + -.. function:: b85encode(b, pad=False) +.. function:: b85encode(b, pad=False, *, wrapcol=0) Encode the :term:`bytes-like object` *b* using base85 (as used in e.g. git-style binary diffs) and return the encoded :class:`bytes`. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + The input is padded with ``b'\0'`` so its length is a multiple of 4 + bytes before encoding. If *pad* is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, and thus the length of the data may not be + preserved on decoding. + + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not add any newlines. .. versionadded:: 3.4 + .. versionchanged:: 3.15 + Added the *wrapcol* parameter. + -.. function:: b85decode(b) +.. function:: b85decode(b, *, ignorechars=b'', canonical=False) Decode the base85-encoded :term:`bytes-like object` or ASCII string *b* and - return the decoded :class:`bytes`. Padding is implicitly removed, if - necessary. + return the decoded :class:`bytes`. + + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + + If *canonical* is true, non-canonical encodings are rejected. + See :func:`binascii.a2b_base85` for details. .. versionadded:: 3.4 + .. versionchanged:: 3.15 + Added the *canonical* and *ignorechars* parameters. + Single-character final groups are now always rejected as encoding + violations. -.. function:: z85encode(s) + +.. function:: z85encode(s, pad=False, *, wrapcol=0) Encode the :term:`bytes-like object` *s* using Z85 (as used in ZeroMQ) - and return the encoded :class:`bytes`. See `Z85 specification - `_ for more information. + and return the encoded :class:`bytes`. + + The input is padded with ``b'\0'`` so its length is a multiple of 4 + bytes before encoding. If *pad* is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, as required by the ZeroMQ standard. + + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not add any newlines. .. versionadded:: 3.13 + .. versionchanged:: 3.15 + The *pad* parameter was added. + + .. versionchanged:: 3.15 + Added the *wrapcol* parameter. + -.. function:: z85decode(s) +.. function:: z85decode(s, *, ignorechars=b'', canonical=False) Decode the Z85-encoded :term:`bytes-like object` or ASCII string *s* and - return the decoded :class:`bytes`. See `Z85 specification - `_ for more information. + return the decoded :class:`bytes`. + + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + + If *canonical* is true, non-canonical encodings are rejected. + See :func:`binascii.a2b_base85` for details. .. versionadded:: 3.13 + .. versionchanged:: 3.15 + Added the *canonical* and *ignorechars* parameters. + Single-character final groups are now always rejected as encoding + violations. + .. _base64-legacy: @@ -353,3 +518,11 @@ recommended to review the security section for any code deployed to production. Section 5.2, "Base64 Content-Transfer-Encoding," provides the definition of the base64 encoding. + `ISO 32000-2 Portable document format - Part 2: PDF 2.0 `_ + Section 7.4.3, "ASCII85Decode Filter," provides the definition + of the Ascii85 encoding used in PDF and PostScript, including + the output character set and the details of data length preservation + using zero-padding and partial output groups. + + `ZeroMQ RFC 32/Z85 `_ + The "Formal Specification" section provides the character set used in Z85. diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index c7a3e0c596b9d03..c8b48901901f98c 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`bdb` module handles basic debugger functions, like setting breakpoints +The :mod:`!bdb` module handles basic debugger functions, like setting breakpoints or managing execution via the debugger. The following exception is defined: @@ -18,7 +18,7 @@ The following exception is defined: Exception raised by the :class:`Bdb` class for quitting the debugger. -The :mod:`bdb` module also defines two classes: +The :mod:`!bdb` module also defines two classes: .. class:: Breakpoint(self, file, line, temporary=False, cond=None, funcname=None) @@ -236,7 +236,7 @@ The :mod:`bdb` module also defines two classes: Normally derived classes don't override the following methods, but they may if they want to redefine the definition of stopping and breakpoints. - .. method:: is_skipped_line(module_name) + .. method:: is_skipped_module(module_name) Return ``True`` if *module_name* matches any skip pattern. diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 1bab785684bbabe..ceb80a35a1a76bb 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -10,10 +10,10 @@ -------------- -The :mod:`binascii` module contains a number of methods to convert between +The :mod:`!binascii` module contains a number of methods to convert between binary and various ASCII-encoded binary representations. Normally, you will not use these functions directly but use wrapper modules like -:mod:`base64` instead. The :mod:`binascii` module contains +:mod:`base64` instead. The :mod:`!binascii` module contains low-level functions written in C for greater speed that are used by the higher-level modules. @@ -28,7 +28,7 @@ higher-level modules. ASCII-only unicode strings are now accepted by the ``a2b_*`` functions. -The :mod:`binascii` module defines the following functions: +The :mod:`!binascii` module defines the following functions: .. function:: a2b_uu(string) @@ -48,34 +48,240 @@ The :mod:`binascii` module defines the following functions: Added the *backtick* parameter. -.. function:: a2b_base64(string, /, *, strict_mode=False) +.. function:: a2b_base64(string, /, *, padded=True, alphabet=BASE64_ALPHABET, strict_mode=False, canonical=False) + a2b_base64(string, /, *, ignorechars, padded=True, alphabet=BASE64_ALPHABET, strict_mode=True, canonical=False) Convert a block of base64 data back to binary and return the binary data. More than one line may be passed at a time. + Optional *alphabet* must be a :class:`bytes` object of length 64 which + specifies an alternative alphabet. + + If *padded* is true, the last group of 4 base 64 alphabet characters must + be padded with the '=' character. + If *padded* is false, padding is neither required nor recognized: + the '=' character is not treated as padding but as a non-alphabet + character, which means it is silently discarded when *strict_mode* is false, + or causes an :exc:`~binascii.Error` when *strict_mode* is true unless + b'=' is included in *ignorechars*. + + If *ignorechars* is specified, it should be a :term:`bytes-like object` + containing characters to ignore from the input when *strict_mode* is true. + If *ignorechars* contains the pad character ``'='``, the pad characters + presented before the end of the encoded data and the excess pad characters + will be ignored. + The default value of *strict_mode* is ``True`` if *ignorechars* is specified, + ``False`` otherwise. + If *strict_mode* is true, only valid base64 data will be converted. Invalid base64 data will raise :exc:`binascii.Error`. Valid base64: - * Conforms to :rfc:`3548`. + * Conforms to :rfc:`4648`. * Contains only characters from the base64 alphabet. * Contains no excess data after padding (including excess padding, newlines, etc.). * Does not start with a padding. + If *canonical* is true, non-zero padding bits in the last group are rejected + with :exc:`binascii.Error`, enforcing canonical encoding as defined in + :rfc:`4648` section 3.5. This check is independent of *strict_mode*. + .. versionchanged:: 3.11 Added the *strict_mode* parameter. + .. versionchanged:: 3.15 + Added the *alphabet*, *canonical*, *ignorechars*, and *padded* parameters. + + +.. function:: b2a_base64(data, *, padded=True, alphabet=BASE64_ALPHABET, wrapcol=0, newline=True) + + Convert binary data to a line(s) of ASCII characters in base64 coding, + as specified in :rfc:`4648`. + + If *padded* is true (default), pad the encoded data with the '=' + character to a size multiple of 4. + If *padded* is false, do not add the pad characters. -.. function:: b2a_base64(data, *, newline=True) + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. - Convert binary data to a line of ASCII characters in base64 coding. The return - value is the converted line, including a newline char if *newline* is - true. The output of this function conforms to :rfc:`3548`. + If *newline* is true (default), a newline character will be added + at the end of the output. .. versionchanged:: 3.6 Added the *newline* parameter. + .. versionchanged:: 3.15 + Added the *alphabet*, *padded* and *wrapcol* parameters. + + +.. function:: a2b_ascii85(string, /, *, foldspaces=False, adobe=False, ignorechars=b'', canonical=False) + + Convert Ascii85 data back to binary and return the binary data. + + Valid Ascii85 data contains characters from the Ascii85 alphabet in groups + of five (except for the final group, which may have from two to five + characters). Each group encodes 32 bits of binary data in the range from + ``0`` to ``2 ** 32 - 1``, inclusive. The special character ``z`` is + accepted as a short form of the group ``!!!!!``, which encodes four + consecutive null bytes. A single-character final group is always rejected + as an encoding violation. + + *foldspaces* is a flag that specifies whether the 'y' short sequence + should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). + This feature is not supported by the "standard" Ascii85 encoding. + + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. If + *adobe* is true, a leading ``<~`` is optionally accepted, while a + trailing ``~>`` is *required*, and :exc:`binascii.Error` is raised + if it is not found. + + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + This should only contain whitespace characters. + + If *canonical* is true, non-canonical encodings are rejected with + :exc:`binascii.Error`. Here "canonical" means the encoding that + :func:`b2a_ascii85` would produce: the ``z`` abbreviation must be used + for all-zero groups (rather than ``!!!!!``), and partial final groups + must use the same padding digits as the encoder. + + Invalid Ascii85 data will raise :exc:`binascii.Error`. + + .. versionadded:: 3.15 + + +.. function:: b2a_ascii85(data, /, *, foldspaces=False, wrapcol=0, pad=False, adobe=False) + + Convert binary data to a formatted sequence of ASCII characters in Ascii85 + coding. The return value is the converted data. + + *foldspaces* is an optional flag that uses the special short sequence 'y' + instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This + feature is not supported by the "standard" Ascii85 encoding. + + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. + + If *pad* is true, the zero-padding applied to the end of the input + is fully retained in the output encoding, as done by ``btoa``, + producing an exact multiple of 5 bytes of output. This is not part + of the standard encoding used in PDF, as it does not preserve the + length of the data. + + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. Note + that while ASCII85Decode streams in PDF documents *must* be + terminated with ``~>``, they *must not* use a leading ``<~``. + + .. versionadded:: 3.15 + + +.. function:: a2b_base85(string, /, *, alphabet=BASE85_ALPHABET, ignorechars=b'', canonical=False) + + Convert Base85 data back to binary and return the binary data. + More than one line may be passed at a time. + + Valid Base85 data contains characters from the Base85 alphabet in groups + of five (except for the final group, which may have from two to five + characters). Each group encodes 32 bits of binary data in the range from + ``0`` to ``2 ** 32 - 1``, inclusive. A single-character final group is + always rejected as an encoding violation. + + Optional *alphabet* must be a :class:`bytes` object of length 85 which + specifies an alternative alphabet. + + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + + If *canonical* is true, non-canonical encodings are rejected with + :exc:`binascii.Error`. Here "canonical" means the encoding that + :func:`b2a_base85` would produce: partial final groups must use the + same padding digits as the encoder. + + Invalid Base85 data will raise :exc:`binascii.Error`. + + .. versionadded:: 3.15 + + +.. function:: b2a_base85(data, /, *, alphabet=BASE85_ALPHABET, wrapcol=0, pad=False) + + Convert binary data to a line of ASCII characters in Base85 coding. + The return value is the converted line. + + Optional *alphabet* must be a :term:`bytes-like object` of length 85 which + specifies an alternative alphabet. + + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. + + If *pad* is true, the zero-padding applied to the end of the input + is retained in the output, which will always be a multiple of 5 + bytes, and thus the length of the data may not be preserved on + decoding. + + .. versionadded:: 3.15 + + +.. function:: a2b_base32(string, /, *, padded=True, alphabet=BASE32_ALPHABET, ignorechars=b'', canonical=False) + + Convert base32 data back to binary and return the binary data. + + Valid base32 data contains characters from the base32 alphabet specified + in :rfc:`4648` in groups of eight (if necessary, the final group is padded + to eight characters with ``=``). Each group encodes 40 bits of binary data + in the range from ``0`` to ``2 ** 40 - 1``, inclusive. + + .. note:: + This function does not map lowercase characters (which are invalid in + standard base32) to their uppercase counterparts, nor does it + contextually map ``0`` to ``O`` and ``1`` to ``I``/``L`` as :rfc:`4648` + allows. + + Optional *alphabet* must be a :class:`bytes` object of length 32 which + specifies an alternative alphabet. + + If *padded* is true, the last group of 8 base 32 alphabet characters must + be padded with the '=' character. + If *padded* is false, the '=' character is treated as other non-alphabet + characters (depending on the value of *ignorechars*). + + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + If *ignorechars* contains the pad character ``'='``, the pad characters + presented before the end of the encoded data and the excess pad characters + will be ignored. + + If *canonical* is true, non-zero padding bits in the last group are rejected + with :exc:`binascii.Error`, enforcing canonical encoding as defined in + :rfc:`4648` section 3.5. + + Invalid base32 data will raise :exc:`binascii.Error`. + + .. versionadded:: 3.15 + +.. function:: b2a_base32(data, /, *, padded=True, alphabet=BASE32_ALPHABET, wrapcol=0) + + Convert binary data to a line of ASCII characters in base32 coding, + as specified in :rfc:`4648`. The return value is the converted line. + + Optional *alphabet* must be a :term:`bytes-like object` of length 32 which + specifies an alternative alphabet. + + If *padded* is true (default), pad the encoded data with the '=' + character to a size multiple of 8. + If *padded* is false, do not add the pad characters. + + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. + + .. versionadded:: 3.15 .. function:: a2b_qp(data, header=False) @@ -150,17 +356,23 @@ The :mod:`binascii` module defines the following functions: .. versionchanged:: 3.8 The *sep* and *bytes_per_sep* parameters were added. -.. function:: a2b_hex(hexstr) - unhexlify(hexstr) +.. function:: a2b_hex(hexstr, *, ignorechars=b'') + unhexlify(hexstr, *, ignorechars=b'') Return the binary data represented by the hexadecimal string *hexstr*. This function is the inverse of :func:`b2a_hex`. *hexstr* must contain an even number of hexadecimal digits (which can be upper or lower case), otherwise an :exc:`Error` exception is raised. - Similar functionality (accepting only text string arguments, but more - liberal towards whitespace) is also accessible using the - :meth:`bytes.fromhex` class method. + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + + Similar functionality (but more liberal towards whitespace) is also accessible + using the :meth:`bytes.fromhex` class method. + + .. versionchanged:: 3.15 + Added the *ignorechars* parameter. + .. exception:: Error @@ -173,6 +385,69 @@ The :mod:`binascii` module defines the following functions: but may be handled by reading a little more data and trying again. +.. data:: BASE64_ALPHABET + + The Base 64 alphabet according to :rfc:`4648`. + + .. versionadded:: 3.15 + +.. data:: URLSAFE_BASE64_ALPHABET + + The "URL and filename safe" Base 64 alphabet according to :rfc:`4648`. + + .. versionadded:: 3.15 + +.. data:: UU_ALPHABET + + The uuencoding alphabet. + + .. versionadded:: 3.15 + +.. data:: CRYPT_ALPHABET + + The Base 64 alphabet used in the :manpage:`crypt(3)` routine and in the GEDCOM format. + + .. versionadded:: 3.15 + +.. data:: BINHEX_ALPHABET + + The Base 64 alphabet used in BinHex 4 (HQX) within the classic Mac OS. + + .. versionadded:: 3.15 + +.. data:: BASE85_ALPHABET + + The Base85 alphabet. + + .. versionadded:: 3.15 + +.. data:: ASCII85_ALPHABET + + The Ascii85 alphabet. + + .. versionadded:: 3.15 + +.. data:: Z85_ALPHABET + + The `Z85 `_ alphabet. + + .. versionadded:: 3.15 + +.. data:: BASE32_ALPHABET + + The Base 32 alphabet according to :rfc:`4648`. + + .. versionadded:: 3.15 + +.. data:: BASE32HEX_ALPHABET + + The "Extended Hex" Base 32 alphabet according to :rfc:`4648`. + Data encoded with this alphabet maintains its sort order during bitwise + comparisons. + + .. versionadded:: 3.15 + + .. seealso:: Module :mod:`base64` diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index 78da563397b6255..39ab75c0904df9c 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -3,9 +3,6 @@ .. module:: bisect :synopsis: Array bisection algorithms for binary searching. -.. sectionauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Raymond Hettinger -.. example based on the PyModules FAQ entry by Aaron Watters **Source code:** :source:`Lib/bisect.py` @@ -16,7 +13,7 @@ having to sort the list after each insertion. For long lists of items with expensive comparison operations, this can be an improvement over linear searches or frequent resorting. -The module is called :mod:`bisect` because it uses a basic bisection +The module is called :mod:`!bisect` because it uses a basic bisection algorithm to do its work. Unlike other bisection tools that search for a specific value, the functions in this module are designed to locate an insertion point. Accordingly, the functions never call an :meth:`~object.__eq__` @@ -24,6 +21,16 @@ method to determine whether a value has been found. Instead, the functions only call the :meth:`~object.__lt__` method and will return an insertion point between values in an array. +.. note:: + + The functions in this module are not thread-safe. If multiple threads + concurrently use :mod:`!bisect` functions on the same sequence, this + may result in undefined behaviour. Likewise, if the provided sequence + is mutated by a different thread while a :mod:`!bisect` function + is operating on it, the result is undefined. For example, using + :py:func:`~bisect.insort_left` on the same list from multiple threads + may result in the list becoming unsorted. + .. _bisect functions: The following functions are provided: @@ -73,7 +80,7 @@ The following functions are provided: Insert *x* in *a* in sorted order. This function first runs :py:func:`~bisect.bisect_left` to locate an insertion point. - Next, it runs the :meth:`!insert` method on *a* to insert *x* at the + Next, it runs the :meth:`~sequence.insert` method on *a* to insert *x* at the appropriate position to maintain sort order. To support inserting records in a table, the *key* function (if any) is @@ -93,7 +100,7 @@ The following functions are provided: entries of *x*. This function first runs :py:func:`~bisect.bisect_right` to locate an insertion point. - Next, it runs the :meth:`!insert` method on *a* to insert *x* at the + Next, it runs the :meth:`~sequence.insert` method on *a* to insert *x* at the appropriate position to maintain sort order. To support inserting records in a table, the *key* function (if any) is @@ -122,7 +129,7 @@ thoughts in mind: they are used. Consequently, if the search functions are used in a loop, the key function may be called again and again on the same array elements. If the key function isn't fast, consider wrapping it with - :py:func:`functools.cache` to avoid duplicate computations. Alternatively, + :py:deco:`functools.cache` to avoid duplicate computations. Alternatively, consider searching an array of precomputed keys to locate the insertion point (as shown in the examples section below). @@ -193,9 +200,9 @@ example uses :py:func:`~bisect.bisect` to look up a letter grade for an exam sco based on a set of ordered numeric breakpoints: 90 and up is an 'A', 80 to 89 is a 'B', and so on:: - >>> def grade(score, breakpoints=[60, 70, 80, 90], grades='FDCBA'): - ... i = bisect(breakpoints, score) - ... return grades[i] + >>> def grade(score): + ... i = bisect([60, 70, 80, 90], score) + ... return "FDCBA"[i] ... >>> [grade(score) for score in [33, 99, 77, 70, 89, 90, 100]] ['F', 'A', 'C', 'C', 'B', 'A', 'A'] diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index ebe2e43febaefa0..6c20e9c94a3eaee 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -4,11 +4,6 @@ .. module:: bz2 :synopsis: Interfaces for bzip2 compression and decompression. -.. moduleauthor:: Gustavo Niemeyer -.. moduleauthor:: Nadeem Vawda -.. sectionauthor:: Gustavo Niemeyer -.. sectionauthor:: Nadeem Vawda - **Source code:** :source:`Lib/bz2.py` -------------- @@ -16,7 +11,7 @@ This module provides a comprehensive interface for compressing and decompressing data using the bzip2 compression algorithm. -The :mod:`bz2` module contains: +The :mod:`!bz2` module contains: * The :func:`.open` function and :class:`BZ2File` class for reading and writing compressed files. @@ -25,6 +20,8 @@ The :mod:`bz2` module contains: * The :func:`compress` and :func:`decompress` functions for one-shot (de)compression. +.. include:: ../includes/optional-module.rst + (De)compression of files ------------------------ @@ -315,7 +312,7 @@ One-shot (de)compression Examples of usage ----------------- -Below are some examples of typical usage of the :mod:`bz2` module. +Below are some examples of typical usage of the :mod:`!bz2` module. Using :func:`compress` and :func:`decompress` to demonstrate round-trip compression: diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index b292d828841f2f3..31faa8c4fb43dc6 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -5,8 +5,6 @@ :synopsis: Functions for working with calendars, including some emulation of the Unix cal program. -.. sectionauthor:: Drew Csillag - **Source code:** :source:`Lib/calendar.py` -------------- @@ -56,13 +54,13 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is .. method:: setfirstweekday(firstweekday) - Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6) + Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6). Identical to setting the :attr:`~Calendar.firstweekday` property. .. method:: iterweekdays() - Return an iterator for the week day numbers that will be used for one + Return an iterator for the weekday numbers that will be used for one week. The first value from the iterator will be the same as the value of the :attr:`~Calendar.firstweekday` property. @@ -88,7 +86,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is Return an iterator for the month *month* in the year *year* similar to :meth:`itermonthdates`, but not restricted by the :class:`datetime.date` range. Days returned will be tuples consisting of a day of the month - number and a week day number. + number and a weekday number. .. method:: itermonthdays3(year, month) @@ -158,6 +156,11 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is :class:`TextCalendar` instances have the following methods: + .. method:: prweek(theweek, width) + + Print a week's calendar as returned by :meth:`formatweek` and without a + final newline. + .. method:: formatday(theday, weekday, width) @@ -405,7 +408,7 @@ For simple text calendars this module provides the following functions. .. function:: monthrange(year, month) - Returns weekday of first day of the month and number of days in month, for the + Returns weekday of first day of the month and number of days in month, for the specified *year* and *month*. @@ -443,11 +446,11 @@ For simple text calendars this module provides the following functions. An unrelated but handy function that takes a time tuple such as returned by the :func:`~time.gmtime` function in the :mod:`time` module, and returns the corresponding Unix timestamp value, assuming an epoch of 1970, and the POSIX - encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each others' + encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each other's inverse. -The :mod:`calendar` module exports the following data attributes: +The :mod:`!calendar` module exports the following data attributes: .. data:: day_name @@ -501,6 +504,14 @@ The :mod:`calendar` module exports the following data attributes: >>> list(calendar.month_name) ['', 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'] + .. caution:: + + In locales with alternative month names forms, the :data:`!month_name` sequence + may not be suitable when a month name stands by itself and not as part of a date. + For instance, in Greek and in many Slavic and Baltic languages, :data:`!month_name` + will produce the month in genitive case. Use :data:`standalone_month_name` for a form + suitable for standalone use. + .. data:: month_abbr @@ -512,6 +523,31 @@ The :mod:`calendar` module exports the following data attributes: >>> list(calendar.month_abbr) ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] + .. caution:: + + In locales with alternative month names forms, the :data:`!month_abbr` sequence + may not be suitable when a month name stands by itself and not as part of a date. + Use :data:`standalone_month_abbr` for a form suitable for standalone use. + + +.. data:: standalone_month_name + + A sequence that represents the months of the year in the current locale + in the standalone form if the locale provides one. Else it is equivalent + to :data:`month_name`. + + .. versionadded:: 3.15 + + +.. data:: standalone_month_abbr + + A sequence that represents the abbreviated months of the year in the current + locale in the standalone form if the locale provides one. Else it is + equivalent to :data:`month_abbr`. + + .. versionadded:: 3.15 + + .. data:: JANUARY FEBRUARY MARCH @@ -540,13 +576,18 @@ The :mod:`calendar` module exports the following data attributes: .. versionadded:: 3.12 -The :mod:`calendar` module defines the following exceptions: +The :mod:`!calendar` module defines the following exceptions: .. exception:: IllegalMonthError(month) - A subclass of :exc:`ValueError`, + A subclass of :exc:`ValueError` and :exc:`IndexError`, raised when the given month number is outside of the range 1-12 (inclusive). + .. versionchanged:: 3.12 + :exc:`IllegalMonthError` is now also a subclass of + :exc:`ValueError`. New code should avoid catching + :exc:`IndexError`. + .. attribute:: month The invalid month number. @@ -579,7 +620,7 @@ Command-line usage .. versionadded:: 2.5 -The :mod:`calendar` module can be executed as a script from the command line +The :mod:`!calendar` module can be executed as a script from the command line to interactively print a calendar. .. code-block:: shell @@ -677,8 +718,7 @@ The following options are accepted: .. option:: month The month of the specified :option:`year` to print the calendar for. - Must be a number between 1 and 12, - and may only be used in text mode. + Must be a number between 1 and 12. Defaults to printing a calendar for the full year. @@ -716,6 +756,11 @@ The following options are accepted: By default, today's date is highlighted in color and can be :ref:`controlled using environment variables `. +.. versionchanged:: 3.15 + By default, the month is now also highlighted in color, and + the days of the week are also in color. This behavior can be + :ref:`controlled using environment variables `. + *HTML-mode options:* .. option:: --css CSS, -c CSS diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index 26518a0458fd813..f602003e49b8218 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -124,7 +124,7 @@ rectangular coordinates to polar coordinates and back. The modulus (absolute value) of a complex number *z* can be computed using the built-in :func:`abs` function. There is no - separate :mod:`cmath` module function for this operation. + separate :mod:`!cmath` module function for this operation. .. function:: polar(z) @@ -338,7 +338,7 @@ Constants .. data:: nan A floating-point "not a number" (NaN) value. Equivalent to - ``float('nan')``. + ``float('nan')``. See also :data:`math.nan`. .. versionadded:: 3.6 @@ -357,7 +357,7 @@ Note that the selection of functions is similar, but not identical, to that in module :mod:`math`. The reason for having two modules is that some users aren't interested in complex numbers, and perhaps don't even know what they are. They would rather have ``math.sqrt(-1)`` raise an exception than return a complex -number. Also note that the functions defined in :mod:`cmath` always return a +number. Also note that the functions defined in :mod:`!cmath` always return a complex number, even if the answer can be expressed as a real number (in which case the complex number has an imaginary part of zero). diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst index 66544f82f6ff3f2..c988fcebd68a019 100644 --- a/Doc/library/cmd.rst +++ b/Doc/library/cmd.rst @@ -4,8 +4,6 @@ .. module:: cmd :synopsis: Build line-oriented command interpreters. -.. sectionauthor:: Eric S. Raymond - **Source code:** :source:`Lib/cmd.py` -------------- @@ -243,9 +241,7 @@ Instances of :class:`Cmd` subclasses have some public instance variables: Cmd Example ----------- -.. sectionauthor:: Raymond Hettinger - -The :mod:`cmd` module is mainly useful for building custom shells that let a +The :mod:`!cmd` module is mainly useful for building custom shells that let a user work with a program interactively. This section presents a simple example of how to build a shell around a few of diff --git a/Doc/library/cmdline.rst b/Doc/library/cmdline.rst index 16c67ddbf7cec28..6418706269f1eda 100644 --- a/Doc/library/cmdline.rst +++ b/Doc/library/cmdline.rst @@ -12,27 +12,28 @@ The following modules have a command-line interface. * :ref:`calendar ` * :mod:`code` * :ref:`compileall ` -* :mod:`cProfile`: see :ref:`profile ` +* ``cProfile``: see :ref:`profiling.tracing ` * :ref:`dis ` * :ref:`doctest ` * :mod:`!encodings.rot_13` -* :mod:`ensurepip` +* :ref:`ensurepip ` * :mod:`filecmp` * :mod:`fileinput` * :mod:`ftplib` * :ref:`gzip ` * :ref:`http.server ` -* :mod:`!idlelib` +* :ref:`idlelib ` * :ref:`inspect ` * :ref:`json ` * :ref:`mimetypes ` -* :mod:`pdb` +* :ref:`pdb ` * :ref:`pickle ` * :ref:`pickletools ` * :ref:`platform ` * :mod:`poplib` -* :ref:`profile ` -* :mod:`pstats` +* :ref:`profiling.sampling ` +* :ref:`profiling.tracing ` +* :ref:`pstats ` * :ref:`py_compile ` * :mod:`pyclbr` * :mod:`pydoc` @@ -52,8 +53,8 @@ The following modules have a command-line interface. * :mod:`turtledemo` * :ref:`unittest ` * :ref:`uuid ` -* :mod:`venv` -* :mod:`webbrowser` +* :ref:`venv ` +* :ref:`webbrowser ` * :ref:`zipapp ` * :ref:`zipfile ` diff --git a/Doc/library/cmdlinelibs.rst b/Doc/library/cmdlinelibs.rst index 085d31af7bca1ff..32f8c2c9f4ae326 100644 --- a/Doc/library/cmdlinelibs.rst +++ b/Doc/library/cmdlinelibs.rst @@ -1,7 +1,7 @@ .. _cmdlinelibs: ******************************** -Command Line Interface Libraries +Command-line interface libraries ******************************** The modules described in this chapter assist with implementing @@ -19,3 +19,4 @@ Here's an overview: curses.rst curses.ascii.rst curses.panel.rst + cmd.rst diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 52587c4dd8f8e87..59c016d21501b03 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -29,7 +29,7 @@ build applications which provide an interactive interpreter prompt. module. -.. class:: InteractiveConsole(locals=None, filename="", local_exit=False) +.. class:: InteractiveConsole(locals=None, filename="", *, local_exit=False) Closely emulate the behavior of the interactive Python interpreter. This class builds on :class:`InteractiveInterpreter` and adds prompting using the familiar diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 1cb0e225bca043a..059ed2c03acfa38 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -4,10 +4,6 @@ .. module:: codecs :synopsis: Encode and decode data and streams. -.. moduleauthor:: Marc-André Lemburg -.. sectionauthor:: Marc-André Lemburg -.. sectionauthor:: Martin v. Löwis - **Source code:** :source:`Lib/codecs.py` .. index:: @@ -68,11 +64,21 @@ The full details for each codec can also be looked up directly: Looks up the codec info in the Python codec registry and returns a :class:`CodecInfo` object as defined below. - Encodings are first looked up in the registry's cache. If not found, the list of + This function first normalizes the *encoding*: all ASCII letters are + converted to lower case, spaces are replaced with hyphens. + Then encoding is looked up in the registry's cache. If not found, the list of registered search functions is scanned. If no :class:`CodecInfo` object is found, a :exc:`LookupError` is raised. Otherwise, the :class:`CodecInfo` object is stored in the cache and returned to the caller. + .. versionchanged:: 3.9 + Any characters except ASCII letters and digits and a dot are converted to underscore. + + .. versionchanged:: 3.15 + No characters are converted to underscore anymore. + Spaces are converted to hyphens. + + .. class:: CodecInfo(encode, decode, streamreader=None, streamwriter=None, incrementalencoder=None, incrementaldecoder=None, name=None) Codec details when looking up the codec registry. The constructor @@ -167,14 +173,11 @@ function: .. function:: register(search_function, /) Register a codec search function. Search functions are expected to take one - argument, being the encoding name in all lower case letters with hyphens - and spaces converted to underscores, and return a :class:`CodecInfo` object. + argument, being the encoding name in all lower case letters with spaces + converted to hyphens, and return a :class:`CodecInfo` object. In case a search function cannot find a given encoding, it should return ``None``. - .. versionchanged:: 3.9 - Hyphens and spaces are converted to underscore. - .. function:: unregister(search_function, /) @@ -310,7 +313,7 @@ and writing to platform dependent files: Codec Base Classes ------------------ -The :mod:`codecs` module defines a set of base classes which define the +The :mod:`!codecs` module defines a set of base classes which define the interfaces for working with codec objects, and can also be used as the basis for custom codec implementations. @@ -982,17 +985,22 @@ defined in Unicode. A simple and straightforward way that can store each Unicode code point, is to store each code point as four consecutive bytes. There are two possibilities: store the bytes in big endian or in little endian order. These two encodings are called ``UTF-32-BE`` and ``UTF-32-LE`` respectively. Their -disadvantage is that if e.g. you use ``UTF-32-BE`` on a little endian machine you -will always have to swap bytes on encoding and decoding. ``UTF-32`` avoids this -problem: bytes will always be in natural endianness. When these bytes are read -by a CPU with a different endianness, then bytes have to be swapped though. To -be able to detect the endianness of a ``UTF-16`` or ``UTF-32`` byte sequence, -there's the so called BOM ("Byte Order Mark"). This is the Unicode character -``U+FEFF``. This character can be prepended to every ``UTF-16`` or ``UTF-32`` -byte sequence. The byte swapped version of this character (``0xFFFE``) is an -illegal character that may not appear in a Unicode text. So when the -first character in a ``UTF-16`` or ``UTF-32`` byte sequence -appears to be a ``U+FFFE`` the bytes have to be swapped on decoding. +disadvantage is that if, for example, you use ``UTF-32-BE`` on a little endian +machine you will always have to swap bytes on encoding and decoding. +Python's ``UTF-16`` and ``UTF-32`` codecs avoid this problem by using the +platform's native byte order when no BOM is present. +Python follows prevailing platform +practice, so native-endian data round-trips without redundant byte swapping, +even though the Unicode Standard defaults to big-endian when the byte order is +unspecified. When these bytes are read by a CPU with a different endianness, +the bytes have to be swapped. To be able to detect the endianness of a +``UTF-16`` or ``UTF-32`` byte sequence, a BOM ("Byte Order Mark") is used. +This is the Unicode character ``U+FEFF``. This character can be prepended to every +``UTF-16`` or ``UTF-32`` byte sequence. The byte swapped version of this character +(``0xFFFE``) is an illegal character that may not appear in a Unicode text. +When the first character of a ``UTF-16`` or ``UTF-32`` byte sequence is +``U+FFFE``, the bytes have to be swapped on decoding. + Unfortunately the character ``U+FEFF`` had a second purpose as a ``ZERO WIDTH NO-BREAK SPACE``: a character that has no width and doesn't allow a word to be split. It can e.g. be used to give hints to a ligature algorithm. @@ -1065,7 +1073,7 @@ or with dictionaries as mapping tables. The following table lists the codecs by name, together with a few common aliases, and the languages for which the encoding is likely used. Neither the list of aliases nor the list of languages is meant to be exhaustive. Notice that spelling alternatives that only differ in -case or use a hyphen instead of an underscore are also valid aliases +case or use a space or a hyphen instead of an underscore are also valid aliases because they are equivalent when normalized by :func:`~encodings.normalize_encoding`. For example, ``'utf-8'`` is a valid alias for the ``'utf_8'`` codec. @@ -1076,7 +1084,7 @@ alias for the ``'utf_8'`` codec. refer to the source :source:`aliases.py ` file. On Windows, ``cpXXX`` codecs are available for all code pages. -But only codecs listed in the following table are guarantead to exist on +But only codecs listed in the following table are guaranteed to exist on other platforms. .. impl-detail:: @@ -1147,7 +1155,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp857 | 857, IBM857 | Turkish | +-----------------+--------------------------------+--------------------------------+ -| cp858 | 858, IBM858 | Western Europe | +| cp858 | 858, IBM00858 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp860 | 860, IBM860 | Portuguese | +-----------------+--------------------------------+--------------------------------+ @@ -1184,7 +1192,7 @@ particular, the following variants typically exist: | | | | | | | .. versionadded:: 3.4 | +-----------------+--------------------------------+--------------------------------+ -| cp1140 | ibm1140 | Western Europe | +| cp1140 | IBM01140 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp1250 | windows-1250 | Central and Eastern Europe | +-----------------+--------------------------------+--------------------------------+ @@ -1251,7 +1259,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | iso8859_3 | iso-8859-3, latin3, L3 | Esperanto, Maltese | +-----------------+--------------------------------+--------------------------------+ -| iso8859_4 | iso-8859-4, latin4, L4 | Baltic languages | +| iso8859_4 | iso-8859-4, latin4, L4 | Northern Europe | +-----------------+--------------------------------+--------------------------------+ | iso8859_5 | iso-8859-5, cyrillic | Belarusian, Bulgarian, | | | | Macedonian, Russian, Serbian | @@ -1332,7 +1340,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | utf_8 | U8, UTF, utf8, cp65001 | all languages | +-----------------+--------------------------------+--------------------------------+ -| utf_8_sig | | all languages | +| utf_8_sig | utf8-sig | all languages | +-----------------+--------------------------------+--------------------------------+ .. versionchanged:: 3.4 @@ -1483,6 +1491,36 @@ to :class:`bytes` mappings. They are not supported by :meth:`bytes.decode` Restoration of the aliases for the binary transforms. +.. _standalone-codec-functions: + +Standalone Codec Functions +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The following functions provide encoding and decoding functionality similar to codecs, +but are not available as named codecs through :func:`codecs.encode` or :func:`codecs.decode`. +They are used internally (for example, by :mod:`pickle`) and behave similarly to the +``string_escape`` codec that was removed in Python 3. + +.. function:: codecs.escape_encode(input, errors=None) + + Encode *input* using escape sequences. Similar to how :func:`repr` on bytes + produces escaped byte values. + + *input* must be a :class:`bytes` object. + + Returns a tuple ``(output, length)`` where *output* is a :class:`bytes` + object and *length* is the number of bytes consumed. + +.. function:: codecs.escape_decode(input, errors=None) + + Decode *input* from escape sequences back to the original bytes. + + *input* must be a :term:`bytes-like object`. + + Returns a tuple ``(output, length)`` where *output* is a :class:`bytes` + object and *length* is the number of bytes consumed. + + .. _text-transforms: Text Transforms @@ -1509,8 +1547,8 @@ mapping. It is not supported by :meth:`str.encode` (which only produces Restoration of the ``rot13`` alias. -:mod:`encodings` --- Encodings package --------------------------------------- +:mod:`!encodings` --- Encodings package +--------------------------------------- .. module:: encodings :synopsis: Encodings package @@ -1569,12 +1607,11 @@ This module implements the following exception: Raised when a codec is invalid or incompatible. -:mod:`encodings.idna` --- Internationalized Domain Names in Applications ------------------------------------------------------------------------- +:mod:`!encodings.idna` --- Internationalized Domain Names in Applications +------------------------------------------------------------------------- .. module:: encodings.idna :synopsis: Internationalized Domain Names implementation -.. moduleauthor:: Martin v. Löwis This module implements :rfc:`3490` (Internationalized Domain Names in Applications) and :rfc:`3492` (Nameprep: A Stringprep Profile for @@ -1612,7 +1649,7 @@ When receiving host names from the wire (such as in reverse name lookup), no automatic conversion to Unicode is performed: applications wishing to present such host names to the user should decode them to Unicode. -The module :mod:`encodings.idna` also implements the nameprep procedure, which +The module :mod:`!encodings.idna` also implements the nameprep procedure, which performs certain normalizations on host names, to achieve case-insensitivity of international domain names, and to unify similar characters. The nameprep functions can be used directly if desired. @@ -1635,8 +1672,8 @@ functions can be used directly if desired. Convert a label to Unicode, as specified in :rfc:`3490`. -:mod:`encodings.mbcs` --- Windows ANSI codepage ------------------------------------------------ +:mod:`!encodings.mbcs` --- Windows ANSI codepage +------------------------------------------------ .. module:: encodings.mbcs :synopsis: Windows ANSI codepage @@ -1653,12 +1690,11 @@ This module implements the ANSI codepage (CP_ACP). Support any error handler. -:mod:`encodings.utf_8_sig` --- UTF-8 codec with BOM signature -------------------------------------------------------------- +:mod:`!encodings.utf_8_sig` --- UTF-8 codec with BOM signature +-------------------------------------------------------------- .. module:: encodings.utf_8_sig :synopsis: UTF-8 codec with BOM signature -.. moduleauthor:: Walter Dörwald This module implements a variant of the UTF-8 codec. On encoding, a UTF-8 encoded BOM will be prepended to the UTF-8 encoded bytes. For the stateful encoder this diff --git a/Doc/library/codeop.rst b/Doc/library/codeop.rst index 16f674adb4b22bb..622e57d2ee63dbd 100644 --- a/Doc/library/codeop.rst +++ b/Doc/library/codeop.rst @@ -4,14 +4,11 @@ .. module:: codeop :synopsis: Compile (possibly incomplete) Python code. -.. sectionauthor:: Moshe Zadka -.. sectionauthor:: Michael Hudson - **Source code:** :source:`Lib/codeop.py` -------------- -The :mod:`codeop` module provides utilities upon which the Python +The :mod:`!codeop` module provides utilities upon which the Python read-eval-print loop can be emulated, as is done in the :mod:`code` module. As a result, you probably don't want to use the module directly; if you want to include such a loop in your program you probably want to use the :mod:`code` @@ -25,7 +22,7 @@ There are two parts to this job: #. Remembering which future statements the user has entered, so subsequent input can be compiled with these in effect. -The :mod:`codeop` module provides a way of doing each of these things, and a way +The :mod:`!codeop` module provides a way of doing each of these things, and a way of doing them both. To do just the former: diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index daa9af6d1dd9c95..10e3790717ed6ed 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -4,9 +4,6 @@ .. module:: collections.abc :synopsis: Abstract base classes for containers -.. moduleauthor:: Raymond Hettinger -.. sectionauthor:: Raymond Hettinger - .. versionadded:: 3.3 Formerly, this module was part of the :mod:`collections` module. @@ -140,6 +137,9 @@ ABC Inherits from Abstract Methods Mi ``__len__``, ``insert`` +:class:`ByteString` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods + ``__len__`` + :class:`Set` :class:`Collection` ``__contains__``, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``, ``__iter__``, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``, ``__len__`` ``__sub__``, ``__rsub__``, ``__xor__``, ``__rxor__`` @@ -260,21 +260,50 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Sequence MutableSequence + ByteString ABCs for read-only and mutable :term:`sequences `. Implementation note: Some of the mixin methods, such as - :meth:`~container.__iter__`, :meth:`~object.__reversed__` and :meth:`index`, make - repeated calls to the underlying :meth:`~object.__getitem__` method. + :meth:`~container.__iter__`, :meth:`~object.__reversed__`, + and :meth:`~sequence.index` make repeated calls to the underlying + :meth:`~object.__getitem__` method. Consequently, if :meth:`~object.__getitem__` is implemented with constant access speed, the mixin methods will have linear performance; however, if the underlying method is linear (as it would be with a linked list), the mixins will have quadratic performance and will likely need to be overridden. - .. versionchanged:: 3.5 - The index() method added support for *stop* and *start* - arguments. + .. method:: index(value, start=0, stop=None) + + Return first index of *value*. + + Raises :exc:`ValueError` if the value is not present. + + Supporting the *start* and *stop* arguments is optional, but recommended. + + .. versionchanged:: 3.5 + The :meth:`~sequence.index` method gained support for + the *stop* and *start* arguments. + + .. deprecated-removed:: 3.12 3.17 + The :class:`ByteString` ABC has been deprecated. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the :ref:`buffer protocol ` at runtime. For use + in type annotations, either use :class:`Buffer` or a union that + explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was + an instance of :class:`!ByteString` never actually told you anything + useful about the object. Other common buffer types such as + :class:`memoryview` were also never understood as subtypes of + :class:`!ByteString` (either at runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. .. class:: Set MutableSet @@ -304,7 +333,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. note:: In CPython, generator-based coroutines (:term:`generators ` - decorated with :func:`@types.coroutine `) are + decorated with :deco:`types.coroutine`) are *awaitables*, even though they do not have an :meth:`~object.__await__` method. Using ``isinstance(gencoro, Awaitable)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. @@ -322,7 +351,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. note:: In CPython, generator-based coroutines (:term:`generators ` - decorated with :func:`@types.coroutine `) are + decorated with :deco:`types.coroutine`) are *awaitables*, even though they do not have an :meth:`~object.__await__` method. Using ``isinstance(gencoro, Coroutine)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. @@ -427,7 +456,7 @@ Notes on using :class:`Set` and :class:`MutableSet` as a mixin: The :class:`Set` mixin provides a :meth:`!_hash` method to compute a hash value for the set; however, :meth:`~object.__hash__` is not defined because not all sets are :term:`hashable` or immutable. To add set hashability using mixins, - inherit from both :meth:`Set` and :meth:`Hashable`, then define + inherit from both :class:`Set` and :class:`Hashable`, then define ``__hash__ = Set._hash``. .. seealso:: diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 5fbdb12f40cafa5..d09a6c92bbd37dc 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -4,9 +4,6 @@ .. module:: collections :synopsis: Container datatypes -.. moduleauthor:: Raymond Hettinger -.. sectionauthor:: Raymond Hettinger - **Source code:** :source:`Lib/collections/__init__.py` .. testsetup:: * @@ -240,7 +237,9 @@ For example:: [('the', 1143), ('and', 966), ('to', 762), ('of', 669), ('i', 631), ('you', 554), ('a', 546), ('my', 514), ('hamlet', 471), ('in', 451)] -.. class:: Counter([iterable-or-mapping]) +.. class:: Counter(**kwargs) + Counter(iterable, /, **kwargs) + Counter(mapping, /, **kwargs) A :class:`Counter` is a :class:`dict` subclass for counting :term:`hashable` objects. It is a collection where elements are stored as dictionary keys @@ -290,7 +289,7 @@ For example:: >>> sorted(c.elements()) ['a', 'a', 'a', 'a', 'b', 'b'] - .. method:: most_common([n]) + .. method:: most_common(n=None) Return a list of the *n* most common elements and their counts from the most common to the least. If *n* is omitted or ``None``, @@ -300,7 +299,9 @@ For example:: >>> Counter('abracadabra').most_common(3) [('a', 5), ('b', 2), ('r', 2)] - .. method:: subtract([iterable-or-mapping]) + .. method:: subtract(**kwargs) + subtract(iterable, /, **kwargs) + subtract(mapping, /, **kwargs) Elements are subtracted from an *iterable* or from another *mapping* (or counter). Like :meth:`dict.update` but subtracts counts instead @@ -325,13 +326,15 @@ For example:: .. versionadded:: 3.10 The usual dictionary methods are available for :class:`Counter` objects - except for two which work differently for counters. + except for these two which work differently for counters: .. method:: fromkeys(iterable) This class method is not implemented for :class:`Counter` objects. - .. method:: update([iterable-or-mapping]) + .. method:: update(**kwargs) + update(iterable, /, **kwargs) + update(mapping, /, **kwargs) Elements are counted from an *iterable* or added-in from another *mapping* (or counter). Like :meth:`dict.update` but adds counts @@ -367,9 +370,11 @@ Several mathematical operations are provided for combining :class:`Counter` objects to produce multisets (counters that have counts greater than zero). Addition and subtraction combine counters by adding or subtracting the counts of corresponding elements. Intersection and union return the minimum and -maximum of corresponding counts. Equality and inclusion compare -corresponding counts. Each operation can accept inputs with signed -counts, but the output will exclude results with counts of zero or less. +maximum of corresponding counts. Symmetric difference returns the difference +between the maximum and minimum of the corresponding counts. Equality and +inclusion compare corresponding counts. Each operation can accept inputs +with signed counts, but the output will exclude results with counts of zero +or below. .. doctest:: @@ -383,6 +388,8 @@ counts, but the output will exclude results with counts of zero or less. Counter({'a': 1, 'b': 1}) >>> c | d # union: max(c[x], d[x]) Counter({'a': 3, 'b': 2}) + >>> c ^ d # max(c[x], d[x]) - min(c[x], d[x]) + Counter({'a': 2, 'b': 1}) >>> c == d # equality: c[x] == d[x] False >>> c <= d # inclusion: c[x] <= d[x] @@ -400,6 +407,9 @@ or subtracting from an empty counter. .. versionadded:: 3.3 Added support for unary plus, unary minus, and in-place multiset operations. +.. versionadded:: 3.15 + Added support for the symmetric difference multiset operation, ``c ^ d``. + .. note:: Counters were primarily designed to work with positive integers to represent @@ -474,17 +484,19 @@ or subtracting from an empty counter. Unix. They are also useful for tracking transactions and other pools of data where only the most recent activity is of interest. + Deques are :ref:`generic ` over the type of their contents. + Deque objects support the following methods: - .. method:: append(x) + .. method:: append(item, /) - Add *x* to the right side of the deque. + Add *item* to the right side of the deque. - .. method:: appendleft(x) + .. method:: appendleft(item, /) - Add *x* to the left side of the deque. + Add *item* to the left side of the deque. .. method:: clear() @@ -499,38 +511,38 @@ or subtracting from an empty counter. .. versionadded:: 3.5 - .. method:: count(x) + .. method:: count(value, /) - Count the number of deque elements equal to *x*. + Count the number of deque elements equal to *value*. .. versionadded:: 3.2 - .. method:: extend(iterable) + .. method:: extend(iterable, /) Extend the right side of the deque by appending elements from the iterable argument. - .. method:: extendleft(iterable) + .. method:: extendleft(iterable, /) Extend the left side of the deque by appending elements from *iterable*. Note, the series of left appends results in reversing the order of elements in the iterable argument. - .. method:: index(x[, start[, stop]]) + .. method:: index(value[, start[, stop]]) - Return the position of *x* in the deque (at or after index *start* + Return the position of *value* in the deque (at or after index *start* and before index *stop*). Returns the first match or raises :exc:`ValueError` if not found. .. versionadded:: 3.5 - .. method:: insert(i, x) + .. method:: insert(index, value, /) - Insert *x* into the deque at position *i*. + Insert *value* into the deque at position *index*. If the insertion would cause a bounded deque to grow beyond *maxlen*, an :exc:`IndexError` is raised. @@ -550,7 +562,7 @@ or subtracting from an empty counter. elements are present, raises an :exc:`IndexError`. - .. method:: remove(value) + .. method:: remove(value, /) Remove the first occurrence of *value*. If not found, raises a :exc:`ValueError`. @@ -563,7 +575,7 @@ or subtracting from an empty counter. .. versionadded:: 3.2 - .. method:: rotate(n=1) + .. method:: rotate(n=1, /) Rotate the deque *n* steps to the right. If *n* is negative, rotate to the left. @@ -715,7 +727,9 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, :class:`defaultdict` objects ---------------------------- -.. class:: defaultdict(default_factory=None, /, [...]) +.. class:: defaultdict(default_factory=None, /, **kwargs) + defaultdict(default_factory, mapping, /, **kwargs) + defaultdict(default_factory, iterable, /, **kwargs) Return a new dictionary-like object. :class:`defaultdict` is a subclass of the built-in :class:`dict` class. It overrides one method and adds one writable @@ -727,11 +741,14 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, as if they were passed to the :class:`dict` constructor, including keyword arguments. + :class:`!defaultdict`\s are :ref:`generic ` over two types, + signifying (respectively) the types of the dictionary's keys and values. + :class:`defaultdict` objects support the following method in addition to the standard :class:`dict` operations: - .. method:: __missing__(key) + .. method:: __missing__(key, /) If the :attr:`default_factory` attribute is ``None``, this raises a :exc:`KeyError` exception with the *key* as argument. @@ -758,9 +775,9 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, .. attribute:: default_factory - This attribute is used by the :meth:`__missing__` method; it is - initialized from the first argument to the constructor, if present, or to - ``None``, if absent. + This attribute is used by the :meth:`~defaultdict.__missing__` method; + it is initialized from the first argument to the constructor, if present, + or to ``None``, if absent. .. versionchanged:: 3.9 Added merge (``|``) and update (``|=``) operators, specified in @@ -783,10 +800,10 @@ sequence of key-value pairs into a dictionary of lists: When each key is encountered for the first time, it is not already in the mapping; so an entry is automatically created using the :attr:`~defaultdict.default_factory` -function which returns an empty :class:`list`. The :meth:`!list.append` +function which returns an empty :class:`list`. The :meth:`list.append` operation then attaches the value to the new list. When keys are encountered again, the look-up proceeds normally (returning the list for that key) and the -:meth:`!list.append` operation adds another value to the list. This technique is +:meth:`list.append` operation adds another value to the list. This technique is simpler and faster than an equivalent technique using :meth:`dict.setdefault`: >>> d = {} @@ -937,7 +954,7 @@ In addition to the methods inherited from tuples, named tuples support three additional methods and two attributes. To prevent conflicts with field names, the method and attribute names start with an underscore. -.. classmethod:: somenamedtuple._make(iterable) +.. classmethod:: somenamedtuple._make(iterable, /) Class method that makes a new instance from an existing sequence or iterable. @@ -1134,7 +1151,9 @@ Some differences from :class:`dict` still remain: * Until Python 3.8, :class:`dict` lacked a :meth:`~object.__reversed__` method. -.. class:: OrderedDict([items]) +.. class:: OrderedDict(**kwargs) + OrderedDict(mapping, /, **kwargs) + OrderedDict(iterable, /, **kwargs) Return an instance of a :class:`dict` subclass that has methods specialized for rearranging dictionary order. @@ -1202,19 +1221,19 @@ If a new entry overwrites an existing entry, the original insertion position is changed and moved to the end:: class LastUpdatedOrderedDict(OrderedDict): - 'Store items in the order the keys were last added' + 'Store items in the order that the keys were last updated.' def __setitem__(self, key, value): super().__setitem__(key, value) self.move_to_end(key) An :class:`OrderedDict` would also be useful for implementing -variants of :func:`functools.lru_cache`: +variants of :deco:`functools.lru_cache`: .. testcode:: from collections import OrderedDict - from time import time + from time import monotonic class TimeBoundedLRU: "LRU Cache that invalidates and refreshes old entries." @@ -1229,10 +1248,10 @@ variants of :func:`functools.lru_cache`: if args in self.cache: self.cache.move_to_end(args) timestamp, result = self.cache[args] - if time() - timestamp <= self.maxage: + if monotonic() - timestamp <= self.maxage: return result result = self.func(*args) - self.cache[args] = time(), result + self.cache[args] = monotonic(), result if len(self.cache) > self.maxsize: self.cache.popitem(last=False) return result @@ -1315,23 +1334,31 @@ subclass directly from :class:`dict`; however, this class can be easier to work with because the underlying dictionary is accessible as an attribute. -.. class:: UserDict([initialdata]) +.. class:: UserDict(**kwargs) + UserDict(mapping, /, **kwargs) + UserDict(iterable, /, **kwargs) Class that simulates a dictionary. The instance's contents are kept in a regular dictionary, which is accessible via the :attr:`data` attribute of - :class:`UserDict` instances. If *initialdata* is provided, :attr:`data` is - initialized with its contents; note that a reference to *initialdata* will not - be kept, allowing it to be used for other purposes. + :class:`!UserDict` instances. If arguments are provided, they are used to + initialize :attr:`data`, like a regular dictionary. In addition to supporting the methods and operations of mappings, - :class:`UserDict` instances provide the following attribute: + :class:`!UserDict` instances provide the following attribute: .. attribute:: data A real dictionary used to store the contents of the :class:`UserDict` class. + :class:`!UserDict` instances also override the following method: + + .. method:: popitem + Remove and return a ``(key, value)`` pair from the wrapped dictionary. Pairs are + returned in the same order as ``data.popitem()``. (For the default + :meth:`dict.popitem`, this order is :abbr:`LIFO (last-in, first-out)`.) If the + dictionary is empty, raises a :exc:`KeyError`. :class:`UserList` objects ------------------------- diff --git a/Doc/library/colorsys.rst b/Doc/library/colorsys.rst index ffebf4e40dd609a..dffc16ae8b7d475 100644 --- a/Doc/library/colorsys.rst +++ b/Doc/library/colorsys.rst @@ -4,13 +4,11 @@ .. module:: colorsys :synopsis: Conversion functions between RGB and other color systems. -.. sectionauthor:: David Ascher - **Source code:** :source:`Lib/colorsys.py` -------------- -The :mod:`colorsys` module defines bidirectional conversions of color values +The :mod:`!colorsys` module defines bidirectional conversions of color values between colors expressed in the RGB (Red Green Blue) color space used in computer monitors and three other coordinate systems: YIQ, HLS (Hue Lightness Saturation) and HSV (Hue Saturation Value). Coordinates in all of these color @@ -24,7 +22,7 @@ spaces, the coordinates are all between 0 and 1. https://poynton.ca/ColorFAQ.html and https://www.cambridgeincolour.com/tutorials/color-spaces.htm. -The :mod:`colorsys` module defines the following functions: +The :mod:`!colorsys` module defines the following functions: .. function:: rgb_to_yiq(r, g, b) diff --git a/Doc/library/compression.rst b/Doc/library/compression.rst index 618b4a3c2bd1705..98719be9992acc4 100644 --- a/Doc/library/compression.rst +++ b/Doc/library/compression.rst @@ -1,6 +1,8 @@ The :mod:`!compression` package =============================== +.. module:: compression + .. versionadded:: 3.14 The :mod:`!compression` package contains the canonical compression modules diff --git a/Doc/library/compression.zstd.rst b/Doc/library/compression.zstd.rst index a901403621b84f8..6d99e36e1e5bb65 100644 --- a/Doc/library/compression.zstd.rst +++ b/Doc/library/compression.zstd.rst @@ -33,6 +33,8 @@ The :mod:`!compression.zstd` module contains: * The :class:`CompressionParameter`, :class:`DecompressionParameter`, and :class:`Strategy` classes for setting advanced (de)compression parameters. +.. include:: ../includes/optional-module.rst + Exceptions ---------- @@ -71,7 +73,7 @@ Reading and writing compressed files argument is not None, a :exc:`!TypeError` will be raised. When writing, the *options* argument can be a dictionary - providing advanced decompression parameters; see + providing advanced compression parameters; see :class:`CompressionParameter` for detailed information about supported parameters. The *level* argument is the compression level to use when writing compressed data. Only one of *level* or *options* may be non-None. @@ -115,7 +117,7 @@ Reading and writing compressed files argument is not None, a :exc:`!TypeError` will be raised. When writing, the *options* argument can be a dictionary - providing advanced decompression parameters; see + providing advanced compression parameters; see :class:`CompressionParameter` for detailed information about supported parameters. The *level* argument is the compression level to use when writing compressed data. Only one of *level* or *options* may be passed. The @@ -329,10 +331,14 @@ Compressing and decompressing data in memory If *max_length* is non-negative, the method returns at most *max_length* bytes of decompressed data. If this limit is reached and further - output can be produced, the :attr:`~.needs_input` attribute will - be set to ``False``. In this case, the next call to + output can be produced (or EOF is reached), the :attr:`~.needs_input` + attribute will be set to ``False``. In this case, the next call to :meth:`~.decompress` may provide *data* as ``b''`` to obtain - more of the output. + more of the output. The full content can thus be read like:: + + process_output(d.decompress(data, max_length)) + while not d.eof and not d.needs_input: + process_output(d.decompress(b"", max_length)) If all of the input data was decompressed and returned (either because this was less than *max_length* bytes, or because diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index dd92765038c4f79..a32c38283134545 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -12,7 +12,7 @@ and :source:`Lib/concurrent/futures/interpreter.py` -------------- -The :mod:`concurrent.futures` module provides a high-level interface for +The :mod:`!concurrent.futures` module provides a high-level interface for asynchronously executing callables. The asynchronous execution can be performed with threads, using @@ -21,6 +21,11 @@ or separate processes, using :class:`ProcessPoolExecutor`. Each implements the same interface, which is defined by the abstract :class:`Executor` class. +:class:`concurrent.futures.Future` must not be confused with +:class:`asyncio.Future`, which is designed for use with :mod:`asyncio` +tasks and coroutines. See the :doc:`asyncio's Future ` +documentation for a detailed comparison of the two. + .. include:: ../includes/wasm-notavail.rst Executor Objects @@ -101,10 +106,10 @@ Executor Objects executor has started running will be completed prior to this method returning. The remaining futures are cancelled. - You can avoid having to call this method explicitly if you use the - :keyword:`with` statement, which will shutdown the :class:`Executor` - (waiting as if :meth:`Executor.shutdown` were called with *wait* set to - ``True``):: + You can avoid having to call this method explicitly if you use the executor + as a :term:`context manager` via the :keyword:`with` statement, which + will shutdown the :class:`Executor` (waiting as if :meth:`Executor.shutdown` + were called with *wait* set to ``True``):: import shutil with ThreadPoolExecutor(max_workers=4) as e: @@ -151,7 +156,9 @@ And:: print(f.result()) executor = ThreadPoolExecutor(max_workers=1) - executor.submit(wait_on_future) + future = executor.submit(wait_on_future) + # Note: calling future.result() would also cause a deadlock because + # the single worker thread is already waiting for wait_on_future(). .. class:: ThreadPoolExecutor(max_workers=None, thread_name_prefix='', initializer=None, initargs=()) @@ -239,6 +246,8 @@ ThreadPoolExecutor Example InterpreterPoolExecutor ----------------------- +.. versionadded:: 3.14 + The :class:`InterpreterPoolExecutor` class uses a pool of interpreters to execute calls asynchronously. It is a :class:`ThreadPoolExecutor` subclass, which means each worker is running in its own thread. @@ -306,7 +315,7 @@ the bytes over a shared :mod:`socket ` or .. note:: The executor may replace uncaught exceptions from *initializer* - with :class:`~concurrent.futures.interpreter.ExecutionFailed`. + with :class:`~concurrent.interpreters.ExecutionFailed`. Other caveats from parent :class:`ThreadPoolExecutor` apply here. @@ -318,11 +327,11 @@ likewise serializes the return value when sending it back. When a worker's current task raises an uncaught exception, the worker always tries to preserve the exception as-is. If that is successful then it also sets the ``__cause__`` to a corresponding -:class:`~concurrent.futures.interpreter.ExecutionFailed` +:class:`~concurrent.interpreters.ExecutionFailed` instance, which contains a summary of the original exception. In the uncommon case that the worker is not able to preserve the original as-is then it directly preserves the corresponding -:class:`~concurrent.futures.interpreter.ExecutionFailed` +:class:`~concurrent.interpreters.ExecutionFailed` instance instead. @@ -342,6 +351,11 @@ that :class:`ProcessPoolExecutor` will not work in the interactive interpreter. Calling :class:`Executor` or :class:`Future` methods from a callable submitted to a :class:`ProcessPoolExecutor` will result in deadlock. +Note that the restrictions on functions and arguments needing to picklable as +per :class:`multiprocessing.Process` apply when using :meth:`~Executor.submit` +and :meth:`~Executor.map` on a :class:`ProcessPoolExecutor`. A function defined +in a REPL or a lambda should not be expected to work. + .. class:: ProcessPoolExecutor(max_workers=None, mp_context=None, initializer=None, initargs=(), max_tasks_per_child=None) An :class:`Executor` subclass that executes calls asynchronously using a pool @@ -372,6 +386,11 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. default in absence of a *mp_context* parameter. This feature is incompatible with the "fork" start method. + .. note:: + Bugs have been reported when using the *max_tasks_per_child* feature that + can result in the :class:`ProcessPoolExecutor` hanging in some + circumstances. Follow its eventual resolution in :gh:`115634`. + .. versionchanged:: 3.3 When one of the worker processes terminates abruptly, a :exc:`~concurrent.futures.process.BrokenProcessPool` error is now raised. @@ -708,15 +727,6 @@ Exception classes .. versionadded:: 3.14 -.. exception:: ExecutionFailed - - Raised from :class:`~concurrent.futures.InterpreterPoolExecutor` when - the given initializer fails or from - :meth:`~concurrent.futures.Executor.submit` when there's an uncaught - exception from the submitted task. - - .. versionadded:: 3.14 - .. currentmodule:: concurrent.futures.process .. exception:: BrokenProcessPool diff --git a/Doc/library/concurrent.interpreters.rst b/Doc/library/concurrent.interpreters.rst index be9d565f8e0d383..a7b115e5f6307dc 100644 --- a/Doc/library/concurrent.interpreters.rst +++ b/Doc/library/concurrent.interpreters.rst @@ -4,9 +4,6 @@ .. module:: concurrent.interpreters :synopsis: Multiple interpreters in the same process -.. moduleauthor:: Eric Snow -.. sectionauthor:: Eric Snow - .. versionadded:: 3.14 **Source code:** :source:`Lib/concurrent/interpreters` @@ -29,12 +26,12 @@ Actual concurrency is available separately through .. seealso:: :class:`~concurrent.futures.InterpreterPoolExecutor` - combines threads with interpreters in a familiar interface. + Combines threads with interpreters in a familiar interface. - .. XXX Add references to the upcoming HOWTO docs in the seealso block. + .. XXX Add references to the upcoming HOWTO docs in the seealso block. :ref:`isolating-extensions-howto` - how to update an extension module to support multiple interpreters + How to update an extension module to support multiple interpreters. :pep:`554` @@ -134,7 +131,7 @@ makes them similar to processes, but they still enjoy in-process efficiency, like threads. All that said, interpreters do naturally support certain flavors of -concurrency, as a powerful side effect of that isolation. +concurrency. There's a powerful side effect of that isolation. It enables a different approach to concurrency than you can take with async or threads. It's a similar concurrency model to CSP or the actor model, diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index bb109a9b742cb77..4d720176fcc334a 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -4,13 +4,6 @@ .. module:: configparser :synopsis: Configuration file parser. -.. moduleauthor:: Ken Manheimer -.. moduleauthor:: Barry Warsaw -.. moduleauthor:: Eric S. Raymond -.. moduleauthor:: Łukasz Langa -.. sectionauthor:: Christopher G. Petrilli -.. sectionauthor:: Łukasz Langa - **Source code:** :source:`Lib/configparser.py` .. index:: @@ -31,6 +24,11 @@ can be customized by end users easily. This library does *not* interpret or write the value-type prefixes used in the Windows Registry extended version of INI syntax. +.. warning:: + Be cautious when parsing data from untrusted sources. A malicious + INI file may cause the decoder to consume considerable CPU and memory + resources. Limiting the size of data to be parsed is recommended. + .. seealso:: Module :mod:`tomllib` @@ -80,7 +78,7 @@ Let's take a very basic configuration file that looks like this: The structure of INI files is described `in the following section <#supported-ini-file-structure>`_. Essentially, the file consists of sections, each of which contains keys with values. -:mod:`configparser` classes can read and write such files. Let's start by +:mod:`!configparser` classes can read and write such files. Let's start by creating the above configuration file programmatically. .. doctest:: @@ -449,7 +447,7 @@ Mapping Protocol Access .. versionadded:: 3.2 Mapping protocol access is a generic name for functionality that enables using -custom objects as if they were dictionaries. In case of :mod:`configparser`, +custom objects as if they were dictionaries. In case of :mod:`!configparser`, the mapping interface implementation is using the ``parser['section']['option']`` notation. @@ -459,7 +457,7 @@ the original parser on demand. What's even more important is that when values are changed on a section proxy, they are actually mutated in the original parser. -:mod:`configparser` objects behave as close to actual dictionaries as possible. +:mod:`!configparser` objects behave as close to actual dictionaries as possible. The mapping interface is complete and adheres to the :class:`~collections.abc.MutableMapping` ABC. However, there are a few differences that should be taken into account: @@ -507,7 +505,7 @@ Customizing Parser Behaviour ---------------------------- There are nearly as many INI format variants as there are applications using it. -:mod:`configparser` goes a long way to provide support for the largest sensible +:mod:`!configparser` goes a long way to provide support for the largest sensible set of INI styles available. The default functionality is mainly dictated by historical background and it's very likely that you will want to customize some of the features. @@ -560,7 +558,7 @@ the :meth:`!__init__` options: * *allow_no_value*, default value: ``False`` Some configuration files are known to include settings without values, but - which otherwise conform to the syntax supported by :mod:`configparser`. The + which otherwise conform to the syntax supported by :mod:`!configparser`. The *allow_no_value* parameter to the constructor can be used to indicate that such values should be accepted: @@ -615,7 +613,7 @@ the :meth:`!__init__` options: prefixes for whole line comments. .. versionchanged:: 3.2 - In previous versions of :mod:`configparser` behaviour matched + In previous versions of :mod:`!configparser` behaviour matched ``comment_prefixes=('#',';')`` and ``inline_comment_prefixes=(';',)``. Please note that config parsers don't support escaping of comment prefixes so @@ -672,7 +670,7 @@ the :meth:`!__init__` options: parsers in new applications. .. versionchanged:: 3.2 - In previous versions of :mod:`configparser` behaviour matched + In previous versions of :mod:`!configparser` behaviour matched ``strict=False``. * *empty_lines_in_values*, default value: ``True`` @@ -842,7 +840,7 @@ be overridden by subclasses or by attribute assignment. Legacy API Examples ------------------- -Mainly because of backwards compatibility concerns, :mod:`configparser` +Mainly because of backwards compatibility concerns, :mod:`!configparser` provides also a legacy API with explicit ``get``/``set`` methods. While there are valid use cases for the methods outlined below, mapping protocol access is preferred for new projects. The legacy API is at times more advanced, @@ -1378,7 +1376,7 @@ Exceptions .. exception:: Error - Base class for all other :mod:`configparser` exceptions. + Base class for all other :mod:`!configparser` exceptions. .. exception:: NoSectionError diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index c0ac4ea8412ebdd..6f005f98bd3ede5 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -22,7 +22,7 @@ A small number of constants live in the built-in namespace. They are: An object frequently used to represent the absence of a value, as when default arguments are not passed to a function. Assignments to ``None`` are illegal and raise a :exc:`SyntaxError`. - ``None`` is the sole instance of the :data:`~types.NoneType` type. + ``None`` is the sole instance of the :class:`~types.NoneType` type. .. data:: NotImplemented @@ -33,7 +33,7 @@ A small number of constants live in the built-in namespace. They are: the other type; may be returned by the in-place binary special methods (e.g. :meth:`~object.__imul__`, :meth:`~object.__iand__`, etc.) for the same purpose. It should not be evaluated in a boolean context. - :data:`!NotImplemented` is the sole instance of the :data:`types.NotImplementedType` type. + :data:`!NotImplemented` is the sole instance of the :class:`types.NotImplementedType` type. .. note:: @@ -65,9 +65,10 @@ A small number of constants live in the built-in namespace. They are: .. index:: single: ...; ellipsis literal .. data:: Ellipsis - The same as the ellipsis literal "``...``". Special value used mostly in conjunction - with extended slicing syntax for user-defined container data types. - ``Ellipsis`` is the sole instance of the :data:`types.EllipsisType` type. + The same as the ellipsis literal "``...``", an object frequently used to + indicate that something is omitted. Assignment to ``Ellipsis`` is possible, but + assignment to ``...`` raises a :exc:`SyntaxError`. + ``Ellipsis`` is the sole instance of the :class:`types.EllipsisType` type. .. data:: __debug__ @@ -97,15 +98,17 @@ should not be used in programs. exit(code=None) Objects that when printed, print a message like "Use quit() or Ctrl-D - (i.e. EOF) to exit", and when called, raise :exc:`SystemExit` with the + (i.e. EOF) to exit", and when accessed directly in the interactive + interpreter or called as functions, raise :exc:`SystemExit` with the specified exit code. .. data:: help :noindex: Object that when printed, prints the message "Type help() for interactive - help, or help(object) for help about object.", and when called, - acts as described :func:`elsewhere `. + help, or help(object) for help about object.", and when accessed directly + in the interactive interpreter, invokes the built-in help system + (see :func:`help`). .. data:: copyright credits diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 176be4ff3339555..666f85997448881 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -21,9 +21,9 @@ Functions and classes provided: .. class:: AbstractContextManager An :term:`abstract base class` for classes that implement - :meth:`object.__enter__` and :meth:`object.__exit__`. A default - implementation for :meth:`object.__enter__` is provided which returns - ``self`` while :meth:`object.__exit__` is an abstract method which by default + :meth:`~object.__enter__` and :meth:`~object.__exit__`. A default + implementation for :meth:`~object.__enter__` is provided which returns + ``self`` while :meth:`~object.__exit__` is an abstract method which by default returns ``None``. See also the definition of :ref:`typecontextmanager`. .. versionadded:: 3.6 @@ -32,9 +32,9 @@ Functions and classes provided: .. class:: AbstractAsyncContextManager An :term:`abstract base class` for classes that implement - :meth:`object.__aenter__` and :meth:`object.__aexit__`. A default - implementation for :meth:`object.__aenter__` is provided which returns - ``self`` while :meth:`object.__aexit__` is an abstract method which by default + :meth:`~object.__aenter__` and :meth:`~object.__aexit__`. A default + implementation for :meth:`~object.__aenter__` is provided which returns + ``self`` while :meth:`~object.__aexit__` is an abstract method which by default returns ``None``. See also the definition of :ref:`async-context-managers`. @@ -88,11 +88,11 @@ Functions and classes provided: the exception has been handled, and execution will resume with the statement immediately following the :keyword:`!with` statement. - :func:`contextmanager` uses :class:`ContextDecorator` so the context managers + :deco:`contextmanager` uses :class:`ContextDecorator` so the context managers it creates can be used as decorators as well as in :keyword:`with` statements. When used as a decorator, a new generator instance is implicitly created on each function call (this allows the otherwise "one-shot" context managers - created by :func:`contextmanager` to meet the requirement that context + created by :deco:`contextmanager` to meet the requirement that context managers support multiple invocations in order to be used as decorators). .. versionchanged:: 3.2 @@ -101,7 +101,7 @@ Functions and classes provided: .. decorator:: asynccontextmanager - Similar to :func:`~contextlib.contextmanager`, but creates an + Similar to :deco:`~contextlib.contextmanager`, but creates an :ref:`asynchronous context manager `. This function is a :term:`decorator` that can be used to define a factory @@ -128,7 +128,7 @@ Functions and classes provided: .. versionadded:: 3.7 - Context managers defined with :func:`asynccontextmanager` can be used + Context managers defined with :deco:`asynccontextmanager` can be used either as decorators or with :keyword:`async with` statements:: import time @@ -148,11 +148,11 @@ Functions and classes provided: When used as a decorator, a new generator instance is implicitly created on each function call. This allows the otherwise "one-shot" context managers - created by :func:`asynccontextmanager` to meet the requirement that context + created by :deco:`asynccontextmanager` to meet the requirement that context managers support multiple invocations in order to be used as decorators. .. versionchanged:: 3.10 - Async context managers created with :func:`asynccontextmanager` can + Async context managers created with :deco:`asynccontextmanager` can be used as decorators. @@ -228,7 +228,7 @@ Functions and classes provided: .. function:: nullcontext(enter_result=None) - Return a context manager that returns *enter_result* from ``__enter__``, but + Return a context manager that returns *enter_result* from :meth:`~object.__enter__`, but otherwise does nothing. It is intended to be used as a stand-in for an optional context manager, for example:: @@ -327,15 +327,15 @@ Functions and classes provided: .. function:: redirect_stdout(new_target) Context manager for temporarily redirecting :data:`sys.stdout` to - another file or file-like object. + another :term:`file object`. This tool adds flexibility to existing functions or classes whose output - is hardwired to stdout. + is hardwired to :data:`sys.stdout`. For example, the output of :func:`help` normally is sent to *sys.stdout*. You can capture that output in a string by redirecting the output to an :class:`io.StringIO` object. The replacement stream is returned from the - ``__enter__`` method and so is available as the target of the + :meth:`~object.__enter__` method and so is available as the target of the :keyword:`with` statement:: with redirect_stdout(io.StringIO()) as f: @@ -366,8 +366,8 @@ Functions and classes provided: .. function:: redirect_stderr(new_target) - Similar to :func:`~contextlib.redirect_stdout` but redirecting - :data:`sys.stderr` to another file or file-like object. + Similar to :func:`~contextlib.redirect_stdout` but redirecting the global + :data:`sys.stderr` to another :term:`file object`. This context manager is :ref:`reentrant `. @@ -396,10 +396,11 @@ Functions and classes provided: A base class that enables a context manager to also be used as a decorator. Context managers inheriting from ``ContextDecorator`` have to implement - ``__enter__`` and ``__exit__`` as normal. ``__exit__`` retains its optional + :meth:`~object.__enter__` and :meth:`~object.__exit__` as normal. + ``__exit__`` retains its optional exception handling even when used as a decorator. - ``ContextDecorator`` is used by :func:`contextmanager`, so you get this + ``ContextDecorator`` is used by :deco:`contextmanager`, so you get this functionality automatically. Example of ``ContextDecorator``:: @@ -466,12 +467,40 @@ Functions and classes provided: statements. If this is not the case, then the original construct with the explicit :keyword:`!with` statement inside the function should be used. + When the decorated callable is a generator function, coroutine function, or + asynchronous generator function, the returned wrapper is of the same kind + and keeps the context manager open for the lifetime of the iteration or + await rather than only for the call that creates the generator or coroutine + object. Wrapped generators and asynchronous generators are explicitly + closed when iteration ends, as if by :func:`closing` or :func:`aclosing`. + + .. note:: + For asynchronous generators the wrapper re-yields each value with + ``async for``; values sent with :meth:`~agen.asend` and exceptions + thrown with :meth:`~agen.athrow` are not forwarded to the wrapped + generator. + .. versionadded:: 3.2 + .. versionchanged:: 3.15 + Decorating a generator function, coroutine function, or asynchronous + generator function now keeps the context manager open across iteration + or await. Previously the context manager exited as soon as the + generator or coroutine object was created. + .. class:: AsyncContextDecorator - Similar to :class:`ContextDecorator` but only for asynchronous functions. + Similar to :class:`ContextDecorator`, but the context manager is entered + and exited with :keyword:`async with`. Decorate coroutine functions and + asynchronous generator functions with this class; the returned wrapper is + of the same kind. + + .. note:: + Synchronous functions and generators are accepted, but the wrapper is + always asynchronous, so the decorated callable must then be awaited or + iterated with ``async for``. If that change of calling convention is + not intended, use :class:`ContextDecorator` instead. Example of ``AsyncContextDecorator``:: @@ -509,6 +538,13 @@ Functions and classes provided: .. versionadded:: 3.10 + .. versionchanged:: 3.15 + Decorating an asynchronous generator function now keeps the context + manager open across iteration. Previously the context manager exited + as soon as the generator object was created. Synchronous functions + and synchronous generator functions are also now accepted, with an + asynchronous wrapper returned. + .. class:: ExitStack() @@ -564,6 +600,10 @@ Functions and classes provided: Raises :exc:`TypeError` instead of :exc:`AttributeError` if *cm* is not a context manager. + .. versionchanged:: 3.15 + Added support for arbitrary descriptors :meth:`!__enter__` and + :meth:`!__exit__`. + .. method:: push(exit) Adds a context manager's :meth:`~object.__exit__` method to the callback stack. @@ -582,6 +622,9 @@ Functions and classes provided: The passed in object is returned from the function, allowing this method to be used as a function decorator. + .. versionchanged:: 3.15 + Added support for arbitrary descriptors :meth:`!__exit__`. + .. method:: callback(callback, /, *args, **kwds) Accepts an arbitrary callback function and arguments and adds it to @@ -639,11 +682,17 @@ Functions and classes provided: Raises :exc:`TypeError` instead of :exc:`AttributeError` if *cm* is not an asynchronous context manager. + .. versionchanged:: 3.15 + Added support for arbitrary descriptors :meth:`!__aenter__` and :meth:`!__aexit__`. + .. method:: push_async_exit(exit) Similar to :meth:`ExitStack.push` but expects either an asynchronous context manager or a coroutine function. + .. versionchanged:: 3.15 + Added support for arbitrary descriptors :meth:`!__aexit__`. + .. method:: push_async_callback(callback, /, *args, **kwds) Similar to :meth:`ExitStack.callback` but expects a coroutine function. @@ -653,7 +702,7 @@ Functions and classes provided: Similar to :meth:`ExitStack.close` but properly handles awaitables. - Continuing the example for :func:`asynccontextmanager`:: + Continuing the example for :deco:`asynccontextmanager`:: async with AsyncExitStack() as stack: connections = [await stack.enter_async_context(get_connection()) @@ -668,7 +717,7 @@ Examples and Recipes -------------------- This section describes some examples and recipes for making effective use of -the tools provided by :mod:`contextlib`. +the tools provided by :mod:`!contextlib`. Supporting a variable number of context managers @@ -697,9 +746,9 @@ context management protocol. Catching exceptions from ``__enter__`` methods ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -It is occasionally desirable to catch exceptions from an ``__enter__`` +It is occasionally desirable to catch exceptions from an :meth:`~object.__enter__` method implementation, *without* inadvertently catching exceptions from -the :keyword:`with` statement body or the context manager's ``__exit__`` +the :keyword:`with` statement body or the context manager's :meth:`~object.__exit__` method. By using :class:`ExitStack` the steps in the context management protocol can be separated slightly in order to allow this:: @@ -911,7 +960,7 @@ Files are an example of effectively single use context managers, since the first :keyword:`with` statement will close the file, preventing any further IO operations using that file object. -Context managers created using :func:`contextmanager` are also single use +Context managers created using :deco:`contextmanager` are also single use context managers, and will complain about the underlying generator failing to yield if an attempt is made to use them a second time:: diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 57580ce026e96ac..b0cc0be8e911bf0 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -4,8 +4,6 @@ .. module:: contextvars :synopsis: Context Variables -.. sectionauthor:: Yury Selivanov - -------------- This module provides APIs to manage, store, and access context-local @@ -44,6 +42,9 @@ Context Variables references to context variables which prevents context variables from being properly garbage collected. + :class:`!ContextVar`\s are :ref:`generic ` over the type of + their contained value. + .. attribute:: ContextVar.name The name of the variable. This is a read-only property. @@ -77,6 +78,32 @@ Context Variables to restore the variable to its previous value via the :meth:`ContextVar.reset` method. + For convenience, the token object can be used as a context manager + to avoid calling :meth:`ContextVar.reset` manually:: + + var = ContextVar('var', default='default value') + + with var.set('new value'): + assert var.get() == 'new value' + + assert var.get() == 'default value' + + It is a shorthand for:: + + var = ContextVar('var', default='default value') + + token = var.set('new value') + try: + assert var.get() == 'new value' + finally: + var.reset(token) + + assert var.get() == 'default value' + + .. versionadded:: 3.14 + + Added support for using tokens as context managers. + .. method:: reset(token) Reset the context variable to the value it had before the @@ -93,24 +120,21 @@ Context Variables # After the reset call the var has no value again, so # var.get() would raise a LookupError. + The same *token* cannot be used twice. + .. class:: Token *Token* objects are returned by the :meth:`ContextVar.set` method. They can be passed to the :meth:`ContextVar.reset` method to revert the value of the variable to what it was before the corresponding - *set*. - - The token supports :ref:`context manager protocol ` - to restore the corresponding context variable value at the exit from - :keyword:`with` block:: - - var = ContextVar('var', default='default value') + *set*. A single token cannot reset a context variable more than once. - with var.set('new value'): - assert var.get() == 'new value' + Tokens support the :ref:`context manager protocol ` + to automatically reset context variables. See :meth:`ContextVar.set`. - assert var.get() == 'default value' + Tokens are :ref:`generic ` over the same type as the + :class:`ContextVar` which created them. .. versionadded:: 3.14 @@ -313,7 +337,7 @@ client:: addr = writer.transport.get_extra_info('socket').getpeername() client_addr_var.set(addr) - # In any code that we call is now possible to get + # In any code that we call, it is now possible to get the # client's address by calling 'client_addr_var.get()'. while True: diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 210ad7188003e66..39fc7800d03a916 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -72,15 +72,19 @@ file, socket, window, or any similar types. It does "copy" functions and classes (shallow and deeply), by returning the original object unchanged; this is compatible with the way these are treated by the :mod:`pickle` module. -Shallow copies of dictionaries can be made using :meth:`dict.copy`, and -of lists by assigning a slice of the entire list, for example, -``copied_list = original_list[:]``. +Shallow copies of many collections can be made using the corresponding +:meth:`!copy` method (such as :meth:`list.copy`, :meth:`dict.copy` or +:meth:`set.copy`), and of sequences (such as lists or bytearrays) by making +a slice of the entire sequence (``sequence[:]``). +However, these methods and slicing can create an instance of the base type +when copying an instance of a subclass, whereas :func:`copy.copy` normally +returns an instance of the same type. .. index:: pair: module; pickle Classes can use the same interfaces to control copying that they use to control pickling. See the description of module :mod:`pickle` for information on these -methods. In fact, the :mod:`copy` module uses the registered +methods. In fact, the :mod:`!copy` module uses the registered pickle functions from the :mod:`copyreg` module. .. index:: diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index 6e3144824ebe91f..d59936029da69df 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -12,7 +12,7 @@ -------------- -The :mod:`copyreg` module offers a way to define functions used while pickling +The :mod:`!copyreg` module offers a way to define functions used while pickling specific objects. The :mod:`pickle` and :mod:`copy` modules use those functions when pickling/copying those objects. The module provides configuration information about object constructors which are not classes. diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst index 9ff37196ccf69ff..647cb4925f31dad 100644 --- a/Doc/library/crypt.rst +++ b/Doc/library/crypt.rst @@ -13,7 +13,7 @@ being deprecated in Python 3.11. The removal was decided in :pep:`594`. Applications can use the :mod:`hashlib` module from the standard library. Other possible replacements are third-party libraries from PyPI: -:pypi:`legacycrypt`, :pypi:`bcrypt`, :pypi:`argon2-cffi`, or :pypi:`passlib`. +:pypi:`legacycrypt`, :pypi:`bcrypt`, or :pypi:`argon2-cffi`. These are not supported or maintained by the Python core team. The last version of Python that provided the :mod:`!crypt` module was diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index d39c4ca4a5838be..21ecdbcc08f3486 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -4,8 +4,6 @@ .. module:: csv :synopsis: Write and read tabular data to and from delimited files. -.. sectionauthor:: Skip Montanaro - **Source code:** :source:`Lib/csv.py` .. index:: @@ -25,14 +23,14 @@ similar enough that it is possible to write a single module which can efficiently manipulate such data, hiding the details of reading and writing the data from the programmer. -The :mod:`csv` module implements classes to read and write tabular data in CSV +The :mod:`!csv` module implements classes to read and write tabular data in CSV format. It allows programmers to say, "write this data in the format preferred by Excel," or "read data from this file which was generated by Excel," without knowing the precise details of the CSV format used by Excel. Programmers can also describe the CSV formats understood by other applications or define their own special-purpose CSV formats. -The :mod:`csv` module's :class:`reader` and :class:`writer` objects read and +The :mod:`!csv` module's :class:`reader` and :class:`writer` objects read and write sequences. Programmers can also read and write data in dictionary form using the :class:`DictReader` and :class:`DictWriter` classes. @@ -47,7 +45,7 @@ using the :class:`DictReader` and :class:`DictWriter` classes. Module Contents --------------- -The :mod:`csv` module defines the following functions: +The :mod:`!csv` module defines the following functions: .. index:: @@ -113,7 +111,7 @@ The :mod:`csv` module defines the following functions: spamwriter.writerow(['Spam', 'Lovely Spam', 'Wonderful Spam']) -.. function:: register_dialect(name[, dialect[, **fmtparams]]) +.. function:: register_dialect(name, /, dialect='excel', **fmtparams) Associate *dialect* with *name*. *name* must be a string. The dialect can be specified either by passing a sub-class of :class:`Dialect`, or @@ -139,13 +137,14 @@ The :mod:`csv` module defines the following functions: Return the names of all registered dialects. -.. function:: field_size_limit([new_limit]) +.. function:: field_size_limit() + field_size_limit(new_limit) Returns the current maximum field size allowed by the parser. If *new_limit* is given, this becomes the new limit. -The :mod:`csv` module defines the following classes: +The :mod:`!csv` module defines the following classes: .. class:: DictReader(f, fieldnames=None, restkey=None, restval=None, \ dialect='excel', *args, **kwds) @@ -294,8 +293,8 @@ The :mod:`csv` module defines the following classes: - the second through n-th rows contain strings where at least one value's length differs from that of the putative header of that column. - Twenty rows after the first row are sampled; if more than half of columns + - rows meet the criteria, :const:`True` is returned. + Twenty-one rows after the header are sampled; if more than half of the + columns + rows meet the criteria, :const:`True` is returned. .. note:: @@ -313,7 +312,7 @@ An example for :class:`Sniffer` use:: .. _csv-constants: -The :mod:`csv` module defines the following constants: +The :mod:`!csv` module defines the following constants: .. data:: QUOTE_ALL @@ -374,7 +373,7 @@ The :mod:`csv` module defines the following constants: .. versionadded:: 3.12 -The :mod:`csv` module defines the following exception: +The :mod:`!csv` module defines the following exception: .. exception:: Error @@ -467,7 +466,8 @@ Dialects support the following attributes: .. attribute:: Dialect.skipinitialspace When :const:`True`, spaces immediately following the *delimiter* are ignored. - The default is :const:`False`. + The default is :const:`False`. When combining ``delimiter=' '`` with + ``skipinitialspace=True``, unquoted empty fields are not allowed. .. attribute:: Dialect.strict @@ -526,7 +526,7 @@ out surrounded by parens. This may cause some problems for other programs which read CSV files (assuming they support complex numbers at all). -.. method:: csvwriter.writerow(row) +.. method:: csvwriter.writerow(row, /) Write the *row* parameter to the writer's file object, formatted according to the current :class:`Dialect`. Return the return value of the call to the @@ -535,7 +535,7 @@ read CSV files (assuming they support complex numbers at all). .. versionchanged:: 3.5 Added support of arbitrary iterables. -.. method:: csvwriter.writerows(rows) +.. method:: csvwriter.writerows(rows, /) Write all elements in *rows* (an iterable of *row* objects as described above) to the writer's file object, formatted according to the current @@ -636,7 +636,7 @@ done:: .. rubric:: Footnotes .. [1] If ``newline=''`` is not specified, newlines embedded inside quoted fields - will not be interpreted correctly, and on platforms that use ``\r\n`` linendings + will not be interpreted correctly, and on platforms that use ``\r\n`` line endings on write an extra ``\r`` will be added. It should always be safe to specify ``newline=''``, since the csv module does its own (:term:`universal `) newline handling. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 09f596101b4d1e1..51f08fdc0544c96 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -4,26 +4,30 @@ .. module:: ctypes :synopsis: A foreign function library for Python. -.. moduleauthor:: Thomas Heller - **Source code:** :source:`Lib/ctypes` -------------- -:mod:`ctypes` is a foreign function library for Python. It provides C compatible +:mod:`!ctypes` is a foreign function library for Python. It provides C compatible data types, and allows calling functions in DLLs or shared libraries. It can be used to wrap these libraries in pure Python. +.. include:: ../includes/optional-module.rst + +.. warning:: + + :mod:`!ctypes` provides low-level access to native libraries and the + process's memory, bypassing Python's safety mechanisms and allowing + execution of arbitrary native code. + Incorrect use can corrupt data and objects, reveal sensitive information, + cause crashes, or otherwise compromise the running process. + .. _ctypes-ctypes-tutorial: ctypes tutorial --------------- -Note: The code samples in this tutorial use :mod:`doctest` to make sure that -they actually work. Since some code samples behave differently under Linux, -Windows, or macOS, they contain doctest directives in comments. - Note: Some code samples reference the ctypes :class:`c_int` type. On platforms where ``sizeof(long) == sizeof(int)`` it is an alias to :class:`c_long`. So, you should not be confused if :class:`c_long` is printed if you would expect @@ -34,13 +38,16 @@ So, you should not be confused if :class:`c_long` is printed if you would expect Loading dynamic link libraries ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` exports the *cdll*, and on Windows *windll* and *oledll* +:mod:`!ctypes` exports the :py:data:`~ctypes.cdll`, and on Windows +:py:data:`~ctypes.windll` and :py:data:`~ctypes.oledll` objects, for loading dynamic link libraries. -You load libraries by accessing them as attributes of these objects. *cdll* -loads libraries which export functions using the standard ``cdecl`` calling -convention, while *windll* libraries call functions using the ``stdcall`` -calling convention. *oledll* also uses the ``stdcall`` calling convention, and +You load libraries by accessing them as attributes of these objects. +:py:data:`!cdll` loads libraries which export functions using the +standard ``cdecl`` calling convention, while :py:data:`!windll` +libraries call functions using the ``stdcall`` +calling convention. +:py:data:`~oledll` also uses the ``stdcall`` calling convention, and assumes the functions return a Windows :c:type:`!HRESULT` error code. The error code is used to automatically raise an :class:`OSError` exception when the function call fails. @@ -70,11 +77,13 @@ Windows appends the usual ``.dll`` file suffix automatically. being used by Python. Where possible, use native Python functionality, or else import and use the ``msvcrt`` module. -On Linux, it is required to specify the filename *including* the extension to +Other systems require the filename *including* the extension to load a library, so attribute access can not be used to load libraries. Either the :meth:`~LibraryLoader.LoadLibrary` method of the dll loaders should be used, -or you should load the library by creating an instance of CDLL by calling -the constructor:: +or you should load the library by creating an instance of :py:class:`CDLL` +by calling the constructor. + +For example, on Linux:: >>> cdll.LoadLibrary("libc.so.6") # doctest: +LINUX @@ -83,7 +92,14 @@ the constructor:: >>> -.. XXX Add section for macOS. +On macOS:: + + >>> cdll.LoadLibrary("libc.dylib") # doctest: +MACOS + + >>> libc = CDLL("libc.dylib") # doctest: +MACOS + >>> libc # doctest: +MACOS + + .. _ctypes-accessing-functions-from-loaded-dlls: @@ -180,7 +196,7 @@ handle (passing ``None`` as single argument to call it with a ``NULL`` pointer): To find out the correct calling convention you have to look into the C header file or the documentation for the function you want to call. -On Windows, :mod:`ctypes` uses win32 structured exception handling to prevent +On Windows, :mod:`!ctypes` uses win32 structured exception handling to prevent crashes from general protection faults when functions are called with invalid argument values:: @@ -190,10 +206,8 @@ argument values:: OSError: exception: access violation reading 0x00000020 >>> -There are, however, enough ways to crash Python with :mod:`ctypes`, so you -should be careful anyway. The :mod:`faulthandler` module can be helpful in -debugging crashes (e.g. from segmentation faults produced by erroneous C library -calls). +The :mod:`faulthandler` module can help debug crashes, +such as segmentation faults produced by erroneous C library calls. ``None``, integers, bytes objects and (unicode) strings are the only native Python objects that can directly be used as parameters in these function calls. @@ -203,7 +217,7 @@ as pointer to the memory block that contains their data (:c:expr:`char *` or :c:expr:`int` type, their value is masked to fit into the C type. Before we move on calling functions with other parameter types, we have to learn -more about :mod:`ctypes` data types. +more about :mod:`!ctypes` data types. .. _ctypes-fundamental-data-types: @@ -211,89 +225,170 @@ more about :mod:`ctypes` data types. Fundamental data types ^^^^^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` defines a number of primitive C compatible data types: - -+----------------------+------------------------------------------+----------------------------+ -| ctypes type | C type | Python type | -+======================+==========================================+============================+ -| :class:`c_bool` | :c:expr:`_Bool` | bool (1) | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_char` | :c:expr:`char` | 1-character bytes object | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_wchar` | :c:type:`wchar_t` | 1-character string | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_byte` | :c:expr:`char` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ubyte` | :c:expr:`unsigned char` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_short` | :c:expr:`short` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ushort` | :c:expr:`unsigned short` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int` | :c:expr:`int` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int8` | :c:type:`int8_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int16` | :c:type:`int16_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int32` | :c:type:`int32_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int64` | :c:type:`int64_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint` | :c:expr:`unsigned int` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint8` | :c:type:`uint8_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint16` | :c:type:`uint16_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint32` | :c:type:`uint32_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint64` | :c:type:`uint64_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_long` | :c:expr:`long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ulong` | :c:expr:`unsigned long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_longlong` | :c:expr:`__int64` or :c:expr:`long long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ulonglong` | :c:expr:`unsigned __int64` or | int | -| | :c:expr:`unsigned long long` | | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_size_t` | :c:type:`size_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ssize_t` | :c:type:`ssize_t` or | int | -| | :c:expr:`Py_ssize_t` | | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_time_t` | :c:type:`time_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_float` | :c:expr:`float` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_double` | :c:expr:`double` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_longdouble`| :c:expr:`long double` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_char_p` | :c:expr:`char *` (NUL terminated) | bytes object or ``None`` | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_wchar_p` | :c:expr:`wchar_t *` (NUL terminated) | string or ``None`` | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_void_p` | :c:expr:`void *` | int or ``None`` | -+----------------------+------------------------------------------+----------------------------+ - -(1) - The constructor accepts any object with a truth value. +:mod:`!ctypes` defines a number of primitive C compatible data types: + +.. list-table:: + :header-rows: 1 + + * - ctypes type + - C type + - Python type + - :py:attr:`~_SimpleCData._type_` + * - :class:`c_bool` + - :c:expr:`_Bool` + - :py:class:`bool` + - ``'?'`` + * - :class:`c_char` + - :c:expr:`char` + - 1-character :py:class:`bytes` + - ``'c'`` + * - :class:`c_wchar` + - :c:type:`wchar_t` + - 1-character :py:class:`str` + - ``'u'`` + * - :class:`c_byte` + - :c:expr:`char` + - :py:class:`int` + - ``'b'`` + * - :class:`c_ubyte` + - :c:expr:`unsigned char` + - :py:class:`int` + - ``'B'`` + * - :class:`c_short` + - :c:expr:`short` + - :py:class:`int` + - ``'h'`` + * - :class:`c_ushort` + - :c:expr:`unsigned short` + - :py:class:`int` + - ``'H'`` + * - :class:`c_int` + - :c:expr:`int` + - :py:class:`int` + - ``'i'`` \* + * - :class:`c_int8` + - :c:type:`int8_t` + - :py:class:`int` + - \* + * - :class:`c_int16` + - :c:type:`int16_t` + - :py:class:`int` + - \* + * - :class:`c_int32` + - :c:type:`int32_t` + - :py:class:`int` + - \* + * - :class:`c_int64` + - :c:type:`int64_t` + - :py:class:`int` + - \* + * - :class:`c_uint` + - :c:expr:`unsigned int` + - :py:class:`int` + - ``'I'`` \* + * - :class:`c_uint8` + - :c:type:`uint8_t` + - :py:class:`int` + - \* + * - :class:`c_uint16` + - :c:type:`uint16_t` + - :py:class:`int` + - \* + * - :class:`c_uint32` + - :c:type:`uint32_t` + - :py:class:`int` + - \* + * - :class:`c_uint64` + - :c:type:`uint64_t` + - :py:class:`int` + - \* + * - :class:`c_long` + - :c:expr:`long` + - :py:class:`int` + - ``'l'`` + * - :class:`c_ulong` + - :c:expr:`unsigned long` + - :py:class:`int` + - ``'L'`` + * - :class:`c_longlong` + - :c:expr:`long long` + - :py:class:`int` + - ``'q'`` \* + * - :class:`c_ulonglong` + - :c:expr:`unsigned long long` + - :py:class:`int` + - ``'Q'`` \* + * - :class:`c_size_t` + - :c:type:`size_t` + - :py:class:`int` + - \* + * - :class:`c_ssize_t` + - :c:type:`Py_ssize_t` + - :py:class:`int` + - \* + * - :class:`c_time_t` + - :c:type:`time_t` + - :py:class:`int` + - \* + * - :class:`c_float` + - :c:expr:`float` + - :py:class:`float` + - ``'f'`` + * - :class:`c_double` + - :c:expr:`double` + - :py:class:`float` + - ``'d'`` + * - :class:`c_longdouble` + - :c:expr:`long double` + - :py:class:`float` + - ``'g'`` \* + * - :class:`c_char_p` + - :c:expr:`char *` (NUL terminated) + - :py:class:`bytes` or ``None`` + - ``'z'`` + * - :class:`c_wchar_p` + - :c:expr:`wchar_t *` (NUL terminated) + - :py:class:`str` or ``None`` + - ``'Z'`` + * - :class:`c_void_p` + - :c:expr:`void *` + - :py:class:`int` or ``None`` + - ``'P'`` + * - :class:`py_object` + - :c:expr:`PyObject *` + - :py:class:`object` + - ``'O'`` + * - :ref:`VARIANT_BOOL ` + - :c:expr:`short int` + - :py:class:`bool` + - ``'v'`` Additionally, if IEC 60559 compatible complex arithmetic (Annex G) is supported in both C and ``libffi``, the following complex types are available: -+----------------------------------+---------------------------------+-----------------+ -| ctypes type | C type | Python type | -+==================================+=================================+=================+ -| :class:`c_float_complex` | :c:expr:`float complex` | complex | -+----------------------------------+---------------------------------+-----------------+ -| :class:`c_double_complex` | :c:expr:`double complex` | complex | -+----------------------------------+---------------------------------+-----------------+ -| :class:`c_longdouble_complex` | :c:expr:`long double complex` | complex | -+----------------------------------+---------------------------------+-----------------+ +.. list-table:: + :header-rows: 1 + + * - ctypes type + - C type + - Python type + - :py:attr:`~_SimpleCData._type_` + * - :class:`c_float_complex` + - :c:expr:`float complex` + - :py:class:`complex` + - ``'Zf'`` + * - :class:`c_double_complex` + - :c:expr:`double complex` + - :py:class:`complex` + - ``'Zd'`` + * - :class:`c_longdouble_complex` + - :c:expr:`long double complex` + - :py:class:`complex` + - ``'Zg'`` + +.. versionchanged:: 3.15 + The :py:attr:`~_SimpleCData._type_` types ``F``, ``D`` and ``G`` have been + replaced with ``Zf``, ``Zd`` and ``Zg``. All these types can be created by calling them with an optional initializer of @@ -307,6 +402,16 @@ the correct type and value:: c_ushort(65533) >>> +The constructors for numeric types will convert input using +:py:meth:`~object.__bool__`, +:py:meth:`~object.__index__` (for ``int``), +:py:meth:`~object.__float__` or :py:meth:`~object.__complex__`. +This means :py:class:`~ctypes.c_bool` accepts any object with a truth value:: + + >>> empty_list = [] + >>> c_bool(empty_list) + c_bool(False) + Since these types are mutable, their value can also be changed afterwards:: >>> i = c_int(42) @@ -395,7 +500,7 @@ from within *IDLE* or *PythonWin*:: >>> As has been mentioned before, all Python types except integers, strings, and -bytes objects have to be wrapped in their corresponding :mod:`ctypes` type, so +bytes objects have to be wrapped in their corresponding :mod:`!ctypes` type, so that they can be converted to the required C data type:: >>> printf(b"An int %d, a double %f\n", 1234, c_double(3.14)) @@ -429,10 +534,10 @@ specify :attr:`~_CFuncPtr.argtypes` for all variadic functions. Calling functions with your own custom data types ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -You can also customize :mod:`ctypes` argument conversion to allow instances of -your own classes be used as function arguments. :mod:`ctypes` looks for an +You can also customize :mod:`!ctypes` argument conversion to allow instances of +your own classes be used as function arguments. :mod:`!ctypes` looks for an :attr:`!_as_parameter_` attribute and uses this as the function argument. The -attribute must be an integer, string, bytes, a :mod:`ctypes` instance, or an +attribute must be an integer, string, bytes, a :mod:`!ctypes` instance, or an object with an :attr:`!_as_parameter_` attribute:: >>> class Bottles: @@ -446,7 +551,7 @@ object with an :attr:`!_as_parameter_` attribute:: >>> If you don't want to store the instance's data in the :attr:`!_as_parameter_` -instance variable, you could define a :class:`property` which makes the +instance variable, you could define a :deco:`property` which makes the attribute available on request. @@ -488,7 +593,7 @@ the Python object passed to the function call, it should do a typecheck or whatever is needed to make sure this object is acceptable, and then return the object itself, its :attr:`!_as_parameter_` attribute, or whatever you want to pass as the C function argument in this case. Again, the result should be an -integer, string, bytes, a :mod:`ctypes` instance, or an object with an +integer, string, bytes, a :mod:`!ctypes` instance, or an object with an :attr:`!_as_parameter_` attribute. @@ -598,7 +703,7 @@ Sometimes a C api function expects a *pointer* to a data type as parameter, probably to write into the corresponding location, or if the data is too large to be passed by value. This is also known as *passing parameters by reference*. -:mod:`ctypes` exports the :func:`byref` function which is used to pass parameters +:mod:`!ctypes` exports the :func:`byref` function which is used to pass parameters by reference. The same effect can be achieved with the :func:`pointer` function, although :func:`pointer` does a lot more work since it constructs a real pointer object, so it is faster to use :func:`byref` if you don't need the pointer @@ -623,12 +728,12 @@ Structures and unions ^^^^^^^^^^^^^^^^^^^^^ Structures and unions must derive from the :class:`Structure` and :class:`Union` -base classes which are defined in the :mod:`ctypes` module. Each subclass must +base classes which are defined in the :mod:`!ctypes` module. Each subclass must define a :attr:`~Structure._fields_` attribute. :attr:`!_fields_` must be a list of *2-tuples*, containing a *field name* and a *field type*. -The field type must be a :mod:`ctypes` type like :class:`c_int`, or any other -derived :mod:`ctypes` type: structure, union, array, pointer. +The field type must be a :mod:`!ctypes` type like :class:`c_int`, or any other +derived :mod:`!ctypes` type: structure, union, array, pointer. Here is a simple example of a POINT structure, which contains two integers named *x* and *y*, and also shows how to initialize a structure in the constructor:: @@ -687,7 +792,7 @@ See :class:`CField`:: .. warning:: - :mod:`ctypes` does not support passing unions or structures with bit-fields + :mod:`!ctypes` does not support passing unions or structures with bit-fields to functions by value. While this may work on 32-bit x86, it's not guaranteed by the library to work in the general case. Unions and structures with bit-fields should always be passed to functions by pointer. @@ -700,16 +805,12 @@ compiler does it. It is possible to override this behavior entirely by specifyi :attr:`~Structure._layout_` class attribute in the subclass definition; see the attribute documentation for details. -It is possible to specify the maximum alignment for the fields by setting -the :attr:`~Structure._pack_` class attribute to a positive integer. -This matches what ``#pragma pack(n)`` does in MSVC. +It is possible to specify the maximum alignment for the fields and/or for the +structure itself by setting the class attributes :attr:`~Structure._pack_` +and/or :attr:`~Structure._align_`, respectively. +See the attribute documentation for details. -It is also possible to set a minimum alignment for how the subclass itself is packed in the -same way ``#pragma align(n)`` works in MSVC. -This can be achieved by specifying a :attr:`~Structure._align_` class attribute -in the subclass definition. - -:mod:`ctypes` uses the native byte order for Structures and Unions. To build +:mod:`!ctypes` uses the native byte order for Structures and Unions. To build structures with non-native byte order, you can use one of the :class:`BigEndianStructure`, :class:`LittleEndianStructure`, :class:`BigEndianUnion`, and :class:`LittleEndianUnion` base classes. These @@ -798,7 +899,7 @@ Pointers ^^^^^^^^ Pointer instances are created by calling the :func:`pointer` function on a -:mod:`ctypes` type:: +:mod:`!ctypes` type:: >>> from ctypes import * >>> i = c_int(42) @@ -812,7 +913,7 @@ returns the object to which the pointer points, the ``i`` object above:: c_long(42) >>> -Note that :mod:`ctypes` does not have OOR (original object return), it constructs a +Note that :mod:`!ctypes` does not have OOR (original object return), it constructs a new, equivalent object each time you retrieve an attribute:: >>> pi.contents is i @@ -856,7 +957,7 @@ item. Behind the scenes, the :func:`pointer` function does more than simply create pointer instances, it has to create pointer *types* first. This is done with the -:func:`POINTER` function, which accepts any :mod:`ctypes` type, and returns a +:func:`POINTER` function, which accepts any :mod:`!ctypes` type, and returns a new type:: >>> PI = POINTER(c_int) @@ -878,7 +979,7 @@ Calling the pointer type without an argument creates a ``NULL`` pointer. False >>> -:mod:`ctypes` checks for ``NULL`` when dereferencing pointers (but dereferencing +:mod:`!ctypes` checks for ``NULL`` when dereferencing pointers (but dereferencing invalid non-\ ``NULL`` pointers would crash Python):: >>> null_ptr[0] @@ -898,7 +999,7 @@ invalid non-\ ``NULL`` pointers would crash Python):: Thread safety without the GIL ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -From Python 3.13 onward, the :term:`GIL` can be disabled on :term:`free threaded ` builds. +From Python 3.13 onward, the :term:`GIL` can be disabled on the :term:`free-threaded build`. In ctypes, reads and writes to a single object concurrently is safe, but not across multiple objects: .. code-block:: pycon @@ -963,7 +1064,7 @@ To set a POINTER type field to ``NULL``, you can assign ``None``:: .. XXX list other conversions... Sometimes you have instances of incompatible types. In C, you can cast one type -into another type. :mod:`ctypes` provides a :func:`cast` function which can be +into another type. :mod:`!ctypes` provides a :func:`cast` function which can be used in the same way. The ``Bar`` structure defined above accepts ``POINTER(c_int)`` pointers or :class:`c_int` arrays for its ``values`` field, but not instances of other types:: @@ -1027,7 +1128,7 @@ work:: >>> because the new ``class cell`` is not available in the class statement itself. -In :mod:`ctypes`, we can define the ``cell`` class and set the +In :mod:`!ctypes`, we can define the ``cell`` class and set the :attr:`~Structure._fields_` attribute later, after the class statement:: >>> from ctypes import * @@ -1061,7 +1162,7 @@ other, and finally follow the pointer chain a few times:: Callback functions ^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` allows creating C callable function pointers from Python callables. +:mod:`!ctypes` allows creating C callable function pointers from Python callables. These are sometimes called *callback functions*. First, you must create a class for the callback function. The class knows the @@ -1160,7 +1261,7 @@ write:: .. note:: Make sure you keep references to :func:`CFUNCTYPE` objects as long as they - are used from C code. :mod:`ctypes` doesn't, and if you don't, they may be + are used from C code. :mod:`!ctypes` doesn't, and if you don't, they may be garbage collected, crashing your program when a callback is made. Also, note that if the callback function is called in a thread created @@ -1179,7 +1280,7 @@ Some shared libraries not only export functions, they also export variables. An example in the Python library itself is the :c:data:`Py_Version`, Python runtime version number encoded in a single constant integer. -:mod:`ctypes` can access values like this with the :meth:`~_CData.in_dll` class methods of +:mod:`!ctypes` can access values like this with the :meth:`~_CData.in_dll` class methods of the type. *pythonapi* is a predefined symbol giving access to the Python C api:: @@ -1198,7 +1299,7 @@ Quoting the docs for that value: tricks with this to provide a dynamically created collection of frozen modules. So manipulating this pointer could even prove useful. To restrict the example -size, we show only how this table can be read with :mod:`ctypes`:: +size, we show only how this table can be read with :mod:`!ctypes`:: >>> from ctypes import * >>> @@ -1244,7 +1345,7 @@ for testing. Try it out with ``import __hello__`` for example. Surprises ^^^^^^^^^ -There are some edges in :mod:`ctypes` where you might expect something other +There are some edges in :mod:`!ctypes` where you might expect something other than what actually happens. Consider the following example:: @@ -1312,7 +1413,7 @@ constructs a new Python object each time! Variable-sized data types ^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` provides some support for variable-sized arrays and structures. +:mod:`!ctypes` provides some support for variable-sized arrays and structures. The :func:`resize` function can be used to resize the memory buffer of an existing ctypes object. The function takes the object as first argument, and @@ -1346,7 +1447,7 @@ get errors accessing other elements:: IndexError: invalid index >>> -Another way to use variable-sized data types with :mod:`ctypes` is to use the +Another way to use variable-sized data types with :mod:`!ctypes` is to use the dynamic nature of Python, and (re-)define the data type after the required size is already known, on a case by case basis. @@ -1356,115 +1457,95 @@ is already known, on a case by case basis. ctypes reference ---------------- - -.. _ctypes-finding-shared-libraries: - -Finding shared libraries -^^^^^^^^^^^^^^^^^^^^^^^^ - -When programming in a compiled language, shared libraries are accessed when -compiling/linking a program, and when the program is run. - -The purpose of the :func:`~ctypes.util.find_library` function is to locate a library in a way -similar to what the compiler or runtime loader does (on platforms with several -versions of a shared library the most recent should be loaded), while the ctypes -library loaders act like when a program is run, and call the runtime loader -directly. - -The :mod:`!ctypes.util` module provides a function which can help to determine -the library to load. - - -.. data:: find_library(name) - :module: ctypes.util - :noindex: - - Try to find a library and return a pathname. *name* is the library name without - any prefix like *lib*, suffix like ``.so``, ``.dylib`` or version number (this - is the form used for the posix linker option :option:`!-l`). If no library can - be found, returns ``None``. - -The exact functionality is system dependent. - -On Linux, :func:`~ctypes.util.find_library` tries to run external programs -(``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the library file. -It returns the filename of the library file. - -.. versionchanged:: 3.6 - On Linux, the value of the environment variable ``LD_LIBRARY_PATH`` is used - when searching for libraries, if a library cannot be found by any other means. - -Here are some examples:: - - >>> from ctypes.util import find_library - >>> find_library("m") - 'libm.so.6' - >>> find_library("c") - 'libc.so.6' - >>> find_library("bz2") - 'libbz2.so.1.0' - >>> - -On macOS and Android, :func:`~ctypes.util.find_library` uses the system's -standard naming schemes and paths to locate the library, and returns a full -pathname if successful:: - - >>> from ctypes.util import find_library - >>> find_library("c") - '/usr/lib/libc.dylib' - >>> find_library("m") - '/usr/lib/libm.dylib' - >>> find_library("bz2") - '/usr/lib/libbz2.dylib' - >>> find_library("AGL") - '/System/Library/Frameworks/AGL.framework/AGL' - >>> - -On Windows, :func:`~ctypes.util.find_library` searches along the system search path, and -returns the full pathname, but since there is no predefined naming scheme a call -like ``find_library("c")`` will fail and return ``None``. - -If wrapping a shared library with :mod:`ctypes`, it *may* be better to determine -the shared library name at development time, and hardcode that into the wrapper -module instead of using :func:`~ctypes.util.find_library` to locate the library at runtime. - - -.. _ctypes-listing-loaded-shared-libraries: - -Listing loaded shared libraries -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When writing code that relies on code loaded from shared libraries, it can be -useful to know which shared libraries have already been loaded into the current -process. - -The :mod:`!ctypes.util` module provides the :func:`~ctypes.util.dllist` function, -which calls the different APIs provided by the various platforms to help determine -which shared libraries have already been loaded into the current process. - -The exact output of this function will be system dependent. On most platforms, -the first entry of this list represents the current process itself, which may -be an empty string. -For example, on glibc-based Linux, the return may look like:: - - >>> from ctypes.util import dllist - >>> dllist() - ['', 'linux-vdso.so.1', '/lib/x86_64-linux-gnu/libm.so.6', '/lib/x86_64-linux-gnu/libc.so.6', ... ] - .. _ctypes-loading-shared-libraries: Loading shared libraries ^^^^^^^^^^^^^^^^^^^^^^^^ There are several ways to load shared libraries into the Python process. One -way is to instantiate one of the following classes: +way is to instantiate :py:class:`CDLL` or one of its subclasses: .. class:: CDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) - Instances of this class represent loaded shared libraries. Functions in these - libraries use the standard C calling convention, and are assumed to return - :c:expr:`int`. + Represents a loaded shared library. + + Functions in this library use the standard C calling convention, and are + assumed to return :c:expr:`int`. + The Python :term:`global interpreter lock` is released before calling any + function exported by these libraries, and reacquired afterwards. + For different function behavior, use a subclass: :py:class:`~ctypes.OleDLL`, + :py:class:`~ctypes.WinDLL`, or :py:class:`~ctypes.PyDLL`. + + If you have an existing :py:attr:`handle ` to an already + loaded shared library, it can be passed as the *handle* argument to wrap + the opened library in a new :py:class:`!CDLL` object. + In this case, *name* is only used to set the :py:attr:`~ctypes.CDLL._name` + attribute, but it may be adjusted and/or validated. + + If *handle* is ``None``, the underlying platform's :manpage:`dlopen(3)` or + `LoadLibraryExW`_ function is used to load the library into + the process, and to get a handle to it. + + *name* is the pathname of the shared library to open. + If *name* does not contain a path separator, the library is found + in a platform-specific way. + + On Windows, the ``.DLL`` suffix may be missing. (For details, see + `LoadLibraryExW`_ documentation.) + Other platform-specific prefixes and suffixes (for example, ``lib``, + ``.so``, ``.dylib``, or version numbers) must be present in *name*; + they are not added automatically. + See :ref:`ctypes-finding-shared-libraries` for more information. + + On non-Windows systems, *name* can be ``None``. In this case, + :c:func:`!dlopen` is called with ``NULL``, which opens the main program + as a "library". + (Some systems do the same is *name* is empty; ``None``/``NULL`` is more + portable.) + + .. admonition:: CPython implementation detail + + Since CPython is linked to ``libc``, a ``None`` *name* is often used + to access the C standard library:: + + >>> printf = ctypes.CDLL(None).printf + >>> printf.argtypes = [ctypes.c_char_p] + >>> printf(b"hello\n") + hello + 6 + + To access the Python C API, prefer :py:data:`ctypes.pythonapi` which + works across platforms. + + The *mode* parameter can be used to specify how the library is loaded. For + details, consult the :manpage:`dlopen(3)` manpage. On Windows, *mode* is + ignored. On posix systems, RTLD_NOW is always added, and is not + configurable. + + The *use_errno* parameter, when set to true, enables a ctypes mechanism that + allows accessing the system :data:`errno` error number in a safe way. + :mod:`!ctypes` maintains a thread-local copy of the system's :data:`errno` + variable; if you call foreign functions created with ``use_errno=True`` then the + :data:`errno` value before the function call is swapped with the ctypes private + copy, the same happens immediately after the function call. + + The function :func:`ctypes.get_errno` returns the value of the ctypes private + copy, and the function :func:`ctypes.set_errno` changes the ctypes private copy + to a new value and returns the former value. + + The *use_last_error* parameter, when set to true, enables the same mechanism for + the Windows error code which is managed by the :func:`GetLastError` and + :func:`!SetLastError` Windows API functions; :func:`ctypes.get_last_error` and + :func:`ctypes.set_last_error` are used to request and change the ctypes private + copy of the windows error code. + + The *winmode* parameter is used on Windows to specify how the library is loaded + (since *mode* is ignored). It takes any value that is valid for the Win32 API + `LoadLibraryExW`_ flags parameter. When omitted, the default is to use the + flags that result in the most secure DLL load, which avoids issues such as DLL + hijacking. Passing the full path to the DLL is the safest way to ensure the + correct library and dependencies are loaded. On Windows creating a :class:`CDLL` instance may fail even if the DLL name exists. When a dependent DLL of the loaded DLL is not found, a @@ -1476,20 +1557,49 @@ way is to instantiate one of the following classes: DLLs and determine which one is not found using Windows debugging and tracing tools. + .. seealso:: + + `Microsoft DUMPBIN tool `_ + -- A tool to find DLL dependents. + + .. versionchanged:: 3.8 + Added *winmode* parameter. + .. versionchanged:: 3.12 The *name* parameter can now be a :term:`path-like object`. -.. seealso:: + Instances of this class have no public methods. Functions exported by the + shared library can be accessed as attributes or by index. Please note that + accessing the function through an attribute caches the result and therefore + accessing it repeatedly returns the same object each time. On the other hand, + accessing it through an index returns a new object each time:: + + >>> from ctypes import CDLL + >>> libc = CDLL("libc.so.6") # On Linux + >>> libc.time == libc.time + True + >>> libc['time'] == libc['time'] + False + + The following public attributes are available. Their name starts with an + underscore to not clash with exported function names: + + .. attribute:: _handle + + The system handle used to access the library. + + .. attribute:: _name + + The name of the library passed in the constructor. - `Microsoft DUMPBIN tool `_ - -- A tool to find DLL dependents. +.. _LoadLibraryExW: https://learn.microsoft.com/en-us/windows/win32/api/libloaderapi/nf-libloaderapi-loadlibraryexw +.. class:: OleDLL -.. class:: OleDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) + See :py:class:`~ctypes.CDLL`, the superclass, for common information. - Instances of this class represent loaded shared libraries, - functions in these libraries use the ``stdcall`` calling convention, and are + Functions in this library use the ``stdcall`` calling convention, and are assumed to return the windows specific :class:`HRESULT` code. :class:`HRESULT` values contain information specifying whether the function call failed or succeeded, together with additional error code. If the return value signals a @@ -1501,133 +1611,51 @@ way is to instantiate one of the following classes: :exc:`WindowsError` used to be raised, which is now an alias of :exc:`OSError`. - .. versionchanged:: 3.12 - - The *name* parameter can now be a :term:`path-like object`. +.. class:: WinDLL -.. class:: WinDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) + See :py:class:`~ctypes.CDLL`, the superclass, for common information. - Instances of this class represent loaded shared libraries, - functions in these libraries use the ``stdcall`` calling convention, and are + Functions in these libraries use the ``stdcall`` calling convention, and are assumed to return :c:expr:`int` by default. .. availability:: Windows - .. versionchanged:: 3.12 - - The *name* parameter can now be a :term:`path-like object`. - - -The Python :term:`global interpreter lock` is released before calling any -function exported by these libraries, and reacquired afterwards. +.. class:: PyDLL + See :py:class:`~ctypes.CDLL`, the superclass, for common information. -.. class:: PyDLL(name, mode=DEFAULT_MODE, handle=None) - - Instances of this class behave like :class:`CDLL` instances, except that the + When functions in this library are called, the Python GIL is *not* released during the function call, and after the function execution the Python error flag is checked. If the error flag is set, a Python exception is raised. - Thus, this is only useful to call Python C api functions directly. - - .. versionchanged:: 3.12 - - The *name* parameter can now be a :term:`path-like object`. - -All these classes can be instantiated by calling them with at least one -argument, the pathname of the shared library. If you have an existing handle to -an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platform's :c:func:`!dlopen` or -:c:func:`!LoadLibrary` function is used to load the library into -the process, and to get a handle to it. - -The *mode* parameter can be used to specify how the library is loaded. For -details, consult the :manpage:`dlopen(3)` manpage. On Windows, *mode* is -ignored. On posix systems, RTLD_NOW is always added, and is not -configurable. - -The *use_errno* parameter, when set to true, enables a ctypes mechanism that -allows accessing the system :data:`errno` error number in a safe way. -:mod:`ctypes` maintains a thread-local copy of the system's :data:`errno` -variable; if you call foreign functions created with ``use_errno=True`` then the -:data:`errno` value before the function call is swapped with the ctypes private -copy, the same happens immediately after the function call. - -The function :func:`ctypes.get_errno` returns the value of the ctypes private -copy, and the function :func:`ctypes.set_errno` changes the ctypes private copy -to a new value and returns the former value. - -The *use_last_error* parameter, when set to true, enables the same mechanism for -the Windows error code which is managed by the :func:`GetLastError` and -:func:`!SetLastError` Windows API functions; :func:`ctypes.get_last_error` and -:func:`ctypes.set_last_error` are used to request and change the ctypes private -copy of the windows error code. - -The *winmode* parameter is used on Windows to specify how the library is loaded -(since *mode* is ignored). It takes any value that is valid for the Win32 API -``LoadLibraryEx`` flags parameter. When omitted, the default is to use the -flags that result in the most secure DLL load, which avoids issues such as DLL -hijacking. Passing the full path to the DLL is the safest way to ensure the -correct library and dependencies are loaded. - -.. versionchanged:: 3.8 - Added *winmode* parameter. + Thus, this is only useful to call Python C API functions directly. .. data:: RTLD_GLOBAL - :noindex: Flag to use as *mode* parameter. On platforms where this flag is not available, it is defined as the integer zero. .. data:: RTLD_LOCAL - :noindex: Flag to use as *mode* parameter. On platforms where this is not available, it is the same as *RTLD_GLOBAL*. .. data:: DEFAULT_MODE - :noindex: The default mode which is used to load shared libraries. On OSX 10.3, this is *RTLD_GLOBAL*, otherwise it is the same as *RTLD_LOCAL*. -Instances of these classes have no public methods. Functions exported by the -shared library can be accessed as attributes or by index. Please note that -accessing the function through an attribute caches the result and therefore -accessing it repeatedly returns the same object each time. On the other hand, -accessing it through an index returns a new object each time:: - - >>> from ctypes import CDLL - >>> libc = CDLL("libc.so.6") # On Linux - >>> libc.time == libc.time - True - >>> libc['time'] == libc['time'] - False - -The following public attributes are available, their name starts with an -underscore to not clash with exported function names: - - -.. attribute:: PyDLL._handle - - The system handle used to access the library. - - -.. attribute:: PyDLL._name - - The name of the library passed in the constructor. Shared libraries can also be loaded by using one of the prefabricated objects, which are instances of the :class:`LibraryLoader` class, either by calling the :meth:`~LibraryLoader.LoadLibrary` method, or by retrieving the library as attribute of the loader instance. - .. class:: LibraryLoader(dlltype) Class which loads shared libraries. *dlltype* should be one of the @@ -1645,44 +1673,36 @@ attribute of the loader instance. These prefabricated library loaders are available: -.. data:: cdll - :noindex: - - Creates :class:`CDLL` instances. + .. data:: cdll + Creates :class:`CDLL` instances. -.. data:: windll - :noindex: - Creates :class:`WinDLL` instances. + .. data:: windll - .. availability:: Windows + Creates :class:`WinDLL` instances. + .. availability:: Windows -.. data:: oledll - :noindex: - Creates :class:`OleDLL` instances. + .. data:: oledll - .. availability:: Windows + Creates :class:`OleDLL` instances. + .. availability:: Windows -.. data:: pydll - :noindex: - Creates :class:`PyDLL` instances. + .. data:: pydll + Creates :class:`PyDLL` instances. -For accessing the C Python api directly, a ready-to-use Python shared library -object is available: -.. data:: pythonapi - :noindex: + .. data:: pythonapi - An instance of :class:`PyDLL` that exposes Python C API functions as - attributes. Note that all these functions are assumed to return C - :c:expr:`int`, which is of course not always the truth, so you have to assign - the correct :attr:`!restype` attribute to use these functions. + An instance of :class:`PyDLL` that exposes Python C API functions as + attributes. Note that all these functions are assumed to return C + :c:expr:`int`, which is of course not always the truth, so you have to assign + the correct :attr:`!restype` attribute to use these functions. .. audit-event:: ctypes.dlopen name ctypes.LibraryLoader @@ -1702,6 +1722,134 @@ object is available: accessing a function raises an auditing event ``ctypes.dlsym/handle`` with arguments ``handle`` (the raw library handle) and ``name``. + +.. _ctypes-finding-shared-libraries: + +Finding shared libraries +^^^^^^^^^^^^^^^^^^^^^^^^ + +When programming in a compiled language, shared libraries are accessed when +compiling/linking a program, and when the program is run. +The programmer specifies a short name; the C compiler, linker, and +runtime dynamic library loader then interact in system-specific ways to find +the filename of the library to load. + +While the mapping from short names to filenames is not consistently exposed +by platforms, the :mod:`!ctypes.util` module provides a function, +:func:`!find_library`, that attempts to match it. +However, as backwards compatibility concerns make it difficult to adjust +its behavior for new platforms and configurations, the function +is :term:`soft deprecated`. + +If wrapping a shared library with :mod:`!ctypes`, consider determining the +shared library name at development time, and hardcoding it into the wrapper +module instead of using :func:`!find_library` to locate the library +at runtime. +Also consider adding a configuration option or environment variable to let +users select a library to use, and then perhaps use :func:`!find_library` +as a default or fallback. + +.. function:: find_library(name) + :module: ctypes.util + + Try to find a library and return a pathname. + + *name* is the "short" library name without any prefix like ``lib``, + suffix like ``.so``, ``.dylib`` or version number. + (This is the form used for the posix linker option :option:`!-l`.) + The result is in a format suitable for passing to :py:class:`~ctypes.CDLL`. + + If no library can be found, return ``None``. + + The exact functionality is system dependent, and is *not guaranteed* + to match the behavior of the compiler, linker, and loader used for + (or by) Python. + It is recommended to only use this function as a default or fallback, + + .. soft-deprecated:: 3.15 + + This function is kept for use in cases where it works, but not expected to + be updated for additional platforms and configurations. + +On Linux, :func:`!find_library` tries to run external +programs (``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the +library file. +If the output of these programs does not correspond to the dynamic +linker used by Python, the result of this function may be misleading. + +.. versionchanged:: 3.6 + On Linux, the value of the environment variable ``LD_LIBRARY_PATH`` is used + when searching for libraries, if a library cannot be found by any other means. + +Here are some examples:: + + >>> from ctypes.util import find_library + >>> find_library("m") + 'libm.so.6' + >>> find_library("c") + 'libc.so.6' + >>> find_library("bz2") + 'libbz2.so.1.0' + >>> + +On macOS and Android, :func:`!find_library` uses the system's +standard naming schemes and paths to locate the library, and returns a full +pathname if successful:: + + >>> from ctypes.util import find_library + >>> find_library("c") + '/usr/lib/libc.dylib' + >>> find_library("m") + '/usr/lib/libm.dylib' + >>> find_library("bz2") + '/usr/lib/libbz2.dylib' + >>> find_library("AGL") + '/System/Library/Frameworks/AGL.framework/AGL' + >>> + +On Windows, :func:`!find_library` searches along the system search path, and +returns the full pathname, but since there is no predefined naming scheme a call +like ``find_library("c")`` will fail and return ``None``. + +.. function:: find_msvcrt() + :module: ctypes.util + + Returns the filename of the VC runtime library used by Python, + and by the extension modules. + + If the name of the library cannot be determined, ``None`` is returned. + Notably, this will happen for recent versions of the VC runtime library, + which are not directly loadable. + + If you need to free memory, for example, allocated by an extension module + with a call to the ``free(void *)``, it is important that you use the + function in the same library that allocated the memory. + + .. availability:: Windows + + +.. _ctypes-listing-loaded-shared-libraries: + +Listing loaded shared libraries +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When writing code that relies on code loaded from shared libraries, it can be +useful to know which shared libraries have already been loaded into the current +process. + +The :mod:`!ctypes.util` module provides the :func:`~ctypes.util.dllist` function, +which calls the different APIs provided by the various platforms to help determine +which shared libraries have already been loaded into the current process. + +The exact output of this function will be system dependent. On most platforms, +the first entry of this list represents the current process itself, which may +be an empty string. +For example, on glibc-based Linux, the return may look like:: + + >>> from ctypes.util import dllist + >>> dllist() + ['', 'linux-vdso.so.1', '/lib/x86_64-linux-gnu/libm.so.6', '/lib/x86_64-linux-gnu/libc.so.6', ... ] + .. _ctypes-foreign-functions: Foreign functions @@ -1928,7 +2076,7 @@ the windows header file is this:: LPCWSTR lpCaption, UINT uType); -Here is the wrapping with :mod:`ctypes`:: +Here is the wrapping with :mod:`!ctypes`:: >>> from ctypes import c_int, WINFUNCTYPE, windll >>> from ctypes.wintypes import HWND, LPCWSTR, UINT @@ -1951,7 +2099,7 @@ function retrieves the dimensions of a specified window by copying them into HWND hWnd, LPRECT lpRect); -Here is the wrapping with :mod:`ctypes`:: +Here is the wrapping with :mod:`!ctypes`:: >>> from ctypes import POINTER, WINFUNCTYPE, windll, WinError >>> from ctypes.wintypes import BOOL, HWND, RECT @@ -1979,7 +2127,7 @@ do the error checking, and raises an exception when the api call failed:: >>> If the :attr:`~_CFuncPtr.errcheck` function returns the argument tuple it receives -unchanged, :mod:`ctypes` continues the normal processing it does on the output +unchanged, :mod:`!ctypes` continues the normal processing it does on the output parameters. If you want to return a tuple of window coordinates instead of a ``RECT`` instance, you can retrieve the fields in the function and return them instead, the normal processing will no longer take place:: @@ -2124,31 +2272,6 @@ Utility functions .. availability:: Windows -.. function:: find_library(name) - :module: ctypes.util - - Try to find a library and return a pathname. *name* is the library name - without any prefix like ``lib``, suffix like ``.so``, ``.dylib`` or version - number (this is the form used for the posix linker option :option:`!-l`). If - no library can be found, returns ``None``. - - The exact functionality is system dependent. - - -.. function:: find_msvcrt() - :module: ctypes.util - - Returns the filename of the VC runtime library used by Python, - and by the extension modules. If the name of the library cannot be - determined, ``None`` is returned. - - If you need to free memory, for example, allocated by an extension module - with a call to the ``free(void *)``, it is important that you use the - function in the same library that allocated the memory. - - .. availability:: Windows - - .. function:: dllist() :module: ctypes.util @@ -2447,10 +2570,32 @@ Fundamental data types Python bytes object or string. When the ``value`` attribute is retrieved from a ctypes instance, usually - a new object is returned each time. :mod:`ctypes` does *not* implement + a new object is returned each time. :mod:`!ctypes` does *not* implement original object return, always a new object is constructed. The same is true for all other ctypes object instances. + Each subclass has a class attribute: + + .. attribute:: _type_ + + Class attribute that contains an internal type code, as a string. + See :ref:`ctypes-fundamental-data-types` for a summary. + + Types marked \* in the summary may be (or always are) aliases of a + different :class:`_SimpleCData` subclass, and will not necessarily + use the listed type code. + For example, if the platform's :c:expr:`long`, :c:expr:`long long` + and :c:expr:`time_t` C types are the same, then :class:`c_long`, + :class:`c_longlong` and :class:`c_time_t` all refer to a single class, + :class:`c_long`, whose :attr:`_type_` code is ``'l'``. + The ``'L'`` code will be unused. + + .. seealso:: + + The :mod:`array` and :ref:`struct ` modules, + as well as third-party modules like `numpy `__, + use similar -- but slightly different -- type codes. + Fundamental data types, when returned as foreign function call results, or, for example, by retrieving structure field members or array items, are transparently @@ -2572,6 +2717,8 @@ These are the fundamental ctypes data types: Represents the C :c:expr:`signed long long` datatype. The constructor accepts an optional integer initializer; no overflow checking is done. + On platforms where ``sizeof(long long) == sizeof(long)`` it is an alias + to :class:`c_long`. .. class:: c_short @@ -2583,11 +2730,15 @@ These are the fundamental ctypes data types: .. class:: c_size_t Represents the C :c:type:`size_t` datatype. + Usually an alias for another unsigned integer type. .. class:: c_ssize_t - Represents the C :c:type:`ssize_t` datatype. + Represents the :c:type:`Py_ssize_t` datatype. + This is a signed version of :c:type:`size_t`; + that is, the POSIX :c:type:`ssize_t` type. + Usually an alias for another integer type. .. versionadded:: 3.2 @@ -2595,6 +2746,7 @@ These are the fundamental ctypes data types: .. class:: c_time_t Represents the C :c:type:`time_t` datatype. + Usually an alias for another integer type. .. versionadded:: 3.12 @@ -2647,6 +2799,8 @@ These are the fundamental ctypes data types: Represents the C :c:expr:`unsigned long long` datatype. The constructor accepts an optional integer initializer; no overflow checking is done. + On platforms where ``sizeof(long long) == sizeof(long)`` it is an alias + to :class:`c_long`. .. class:: c_ushort @@ -2698,8 +2852,11 @@ These are the fundamental ctypes data types: .. versionchanged:: 3.14 :class:`!py_object` is now a :term:`generic type`. +.. _ctypes-wintypes: + The :mod:`!ctypes.wintypes` module provides quite some other Windows specific -data types, for example :c:type:`!HWND`, :c:type:`!WPARAM`, or :c:type:`!DWORD`. +data types, for example :c:type:`!HWND`, :c:type:`!WPARAM`, +:c:type:`!VARIANT_BOOL` or :c:type:`!DWORD`. Some useful structures like :c:type:`!MSG` or :c:type:`!RECT` are also defined. @@ -2746,7 +2903,7 @@ fields, or any other data types containing pointer type fields. Abstract base class for structures in *native* byte order. Concrete structure and union types must be created by subclassing one of these - types, and at least define a :attr:`_fields_` class variable. :mod:`ctypes` will + types, and at least define a :attr:`_fields_` class variable. :mod:`!ctypes` will create :term:`descriptor`\s which allow reading and writing the fields by direct attribute accesses. These are the @@ -2792,11 +2949,18 @@ fields, or any other data types containing pointer type fields. .. attribute:: _pack_ An optional small integer that allows overriding the alignment of - structure fields in the instance. :attr:`_pack_` must already be defined - when :attr:`_fields_` is assigned, otherwise it will have no effect. - Setting this attribute to 0 is the same as not setting it at all. + structure fields in the instance. + + This is only implemented for the MSVC-compatible memory layout + (see :attr:`_layout_`). + + Setting :attr:`!_pack_` to 0 is the same as not setting it at all. + Otherwise, the value must be a positive power of two. + The effect is equivalent to ``#pragma pack(N)`` in C, except + :mod:`!ctypes` may allow larger *n* than what the compiler accepts. - This is only implemented for the MSVC-compatible memory layout. + :attr:`!_pack_` must already be defined + when :attr:`_fields_` is assigned, otherwise it will have no effect. .. deprecated-removed:: 3.14 3.19 @@ -2809,9 +2973,22 @@ fields, or any other data types containing pointer type fields. .. attribute:: _align_ - An optional small integer that allows overriding the alignment of + An optional small integer that allows increasing the alignment of the structure when being packed or unpacked to/from memory. - Setting this attribute to 0 is the same as not setting it at all. + + The value must not be negative. + The effect is equivalent to ``__attribute__((aligned(N)))`` on GCC + or ``#pragma align(N)`` on MSVC, except :mod:`!ctypes` may allow + values that the compiler would reject. + + :attr:`!_align_` can only *increase* a structure's alignment + requirements. Setting it to 0 or 1 has no effect. + + Using values that are not powers of two is discouraged and may lead to + surprising behavior. + + :attr:`!_align_` must already be defined + when :attr:`_fields_` is assigned, otherwise it will have no effect. .. versionadded:: 3.13 @@ -2850,7 +3027,7 @@ fields, or any other data types containing pointer type fields. assigned, otherwise it will have no effect. The fields listed in this variable must be structure or union type fields. - :mod:`ctypes` will create descriptors in the structure type that allows + :mod:`!ctypes` will create descriptors in the structure type that allows accessing the nested fields directly, without the need to create the structure or union field. @@ -2994,12 +3171,14 @@ Arrays and pointers Abstract base class for arrays. The recommended way to create concrete array types is by multiplying any - :mod:`ctypes` data type with a non-negative integer. Alternatively, you can subclass + :mod:`!ctypes` data type with a non-negative integer. Alternatively, you can subclass this type and define :attr:`_length_` and :attr:`_type_` class variables. Array elements can be read and written using standard subscript and slice accesses; for slice reads, the resulting object is *not* itself an :class:`Array`. + Arrays are :ref:`generic ` over the type of their elements. + .. attribute:: _length_ @@ -3020,10 +3199,10 @@ Arrays and pointers Create an array. Equivalent to ``type * length``, where *type* is a - :mod:`ctypes` data type and *length* an integer. + :mod:`!ctypes` data type and *length* an integer. - This function is :term:`soft deprecated` in favor of multiplication. - There are no plans to remove it. + .. soft-deprecated:: 3.14 + In favor of multiplication. .. class:: _Pointer diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst index cb895664ff1b11c..3ac76edd38dcc8f 100644 --- a/Doc/library/curses.ascii.rst +++ b/Doc/library/curses.ascii.rst @@ -4,14 +4,11 @@ .. module:: curses.ascii :synopsis: Constants and set-membership functions for ASCII characters. -.. moduleauthor:: Eric S. Raymond -.. sectionauthor:: Eric S. Raymond - **Source code:** :source:`Lib/curses/ascii.py` -------------- -The :mod:`curses.ascii` module supplies name constants for ASCII characters and +The :mod:`!curses.ascii` module supplies name constants for ASCII characters and functions to test membership in various ASCII character classes. The constants supplied are names for control characters as follows: @@ -117,7 +114,7 @@ C library: .. function:: isblank(c) - Checks for an ASCII whitespace character; space or horizontal tab. + Checks for an ASCII blank character; space or horizontal tab. .. function:: iscntrl(c) @@ -133,7 +130,7 @@ C library: .. function:: isgraph(c) - Checks for ASCII any printable character except space. + Checks for any ASCII printable character except space. .. function:: islower(c) @@ -148,7 +145,7 @@ C library: .. function:: ispunct(c) - Checks for any printable ASCII character which is not a space or an alphanumeric + Checks for any ASCII printable character which is not a space or an alphanumeric character. @@ -171,7 +168,8 @@ C library: .. function:: isctrl(c) - Checks for an ASCII control character (ordinal values 0 to 31). + Checks for an ASCII control character (ordinal values 0 to 31). Unlike + :func:`iscntrl`, this does not include the delete character (0x7f). .. function:: ismeta(c) diff --git a/Doc/library/curses.panel.rst b/Doc/library/curses.panel.rst index 11fd841d381f69f..2cfd522f34879a7 100644 --- a/Doc/library/curses.panel.rst +++ b/Doc/library/curses.panel.rst @@ -4,8 +4,6 @@ .. module:: curses.panel :synopsis: A panel stack extension that adds depth to curses windows. -.. sectionauthor:: A.M. Kuchling - -------------- Panels are windows with the added feature of depth, so they can be stacked on @@ -18,7 +16,7 @@ displayed. Panels can be added, moved up or down in the stack, and removed. Functions --------- -The module :mod:`curses.panel` defines the following functions: +The module :mod:`!curses.panel` defines the following functions: .. function:: bottom_panel() @@ -28,7 +26,8 @@ The module :mod:`curses.panel` defines the following functions: .. function:: new_panel(win) - Returns a panel object, associating it with the given window *win*. Be aware + Returns a panel object, associating it with the given window *win* and + placing the new panel on top of the panel stack. Be aware that you need to keep the returned panel object referenced explicitly. If you don't, the panel object is garbage collected and removed from the panel stack. @@ -46,7 +45,7 @@ The module :mod:`curses.panel` defines the following functions: .. _curses-panel-objects: -Panel Objects +Panel objects ------------- Panel objects, as returned by :func:`new_panel` above, are windows with a @@ -101,7 +100,8 @@ Panel objects have the following methods: .. method:: Panel.show() - Display the panel (which might have been hidden). + Display the panel (which might have been hidden), placing it on top of + the panel stack. .. method:: Panel.top() diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 0b13c559295f3c6..d7873054d6b9154 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -4,16 +4,12 @@ .. module:: curses :synopsis: An interface to the curses library, providing portable terminal handling. - :platform: Unix - -.. sectionauthor:: Moshe Zadka -.. sectionauthor:: Eric Raymond **Source code:** :source:`Lib/curses` -------------- -The :mod:`curses` module provides an interface to the curses library, the +The :mod:`!curses` module provides an interface to the curses library, the de-facto standard for portable advanced terminal handling. While curses is most widely used in the Unix environment, versions are available @@ -23,6 +19,10 @@ Linux and the BSD variants of Unix. .. include:: ../includes/wasm-mobile-notavail.rst +.. include:: ../includes/optional-module.rst + +.. availability:: Unix. + .. note:: Whenever the documentation mentions a *character* it can be specified @@ -52,7 +52,7 @@ Linux and the BSD variants of Unix. Functions --------- -The module :mod:`curses` defines the following exception: +The module :mod:`!curses` defines the following exception: .. exception:: error @@ -65,7 +65,7 @@ The module :mod:`curses` defines the following exception: default to the current cursor location. Whenever *attr* is optional, it defaults to :const:`A_NORMAL`. -The module :mod:`curses` defines the following functions: +The module :mod:`!curses` defines the following functions: .. function:: assume_default_colors(fg, bg, /) @@ -80,6 +80,8 @@ The module :mod:`curses` defines the following functions: * Change the definition of the color-pair ``0`` to ``(fg, bg)``. + This is an ncurses extension. + .. versionadded:: 3.14 @@ -116,7 +118,8 @@ The module :mod:`curses` defines the following functions: Return the intensity of the red, green, and blue (RGB) components in the color *color_number*, which must be between ``0`` and ``COLORS - 1``. Return a 3-tuple, containing the R,G,B values for the given color, which will be between - ``0`` (no component) and ``1000`` (maximum amount of component). + ``0`` (no component) and ``1000`` (maximum amount of component). Raise an + exception if the color is not supported. .. function:: color_pair(pair_number) @@ -162,7 +165,7 @@ The module :mod:`curses` defines the following functions: Update the physical screen. The curses library keeps two data structures, one representing the current physical screen contents and a virtual screen - representing the desired next state. The :func:`doupdate` ground updates the + representing the desired next state. The :func:`doupdate` function updates the physical screen to match the virtual screen. The virtual screen may be updated by a :meth:`~window.noutrefresh` call after write @@ -194,13 +197,24 @@ The module :mod:`curses` defines the following functions: .. function:: filter() The :func:`.filter` routine, if used, must be called before :func:`initscr` is - called. The effect is that, during those calls, :envvar:`LINES` is set to ``1``; the + called. The effect is that, during the initialization, :envvar:`LINES` is set to ``1``; the capabilities ``clear``, ``cup``, ``cud``, ``cud1``, ``cuu1``, ``cuu``, ``vpa`` are disabled; and the ``home`` string is set to the value of ``cr``. The effect is that the cursor is confined to the current line, and so are screen updates. This may be used for enabling character-at-a-time line editing without touching the rest of the screen. +.. function:: nofilter() + + Undo the effect of a previous :func:`.filter` call. + Like :func:`.filter`, it must be called before :func:`initscr` so that the + next initialization uses the full screen again. + + Availability: if the underlying curses library provides ``nofilter()``. + + .. versionadded:: next + + .. function:: flash() Flash the screen. That is, change it to reverse-video and then change it back @@ -240,9 +254,10 @@ The module :mod:`curses` defines the following functions: .. function:: getwin(file) - Read window related data stored in the file by an earlier :func:`window.putwin` call. + Read window-related data stored in the file by an earlier :meth:`window.putwin` call. The routine then creates and initializes a new window using that data, returning - the new window object. + the new window object. The *file* argument must be a file object opened for + reading in binary mode. .. function:: has_colors() @@ -253,7 +268,7 @@ The module :mod:`curses` defines the following functions: Return ``True`` if the module supports extended colors; otherwise, return ``False``. Extended color support allows more than 256 color pairs for - terminals that support more than 16 colors (e.g. xterm-256color). + terminals that support more than 16 colors (for example, xterm-256color). Extended color support requires ncurses version 6.1 or later. @@ -326,6 +341,13 @@ The module :mod:`curses` defines the following functions: cause the interpreter to exit. +.. function:: intrflush(flag) + + If *flag* is ``True``, pressing an interrupt key (interrupt, break, or quit) + will flush all output in the terminal driver queue. If *flag* is ``False``, + no flushing is done. + + .. function:: is_term_resized(nlines, ncols) Return ``True`` if :func:`resize_term` would modify the window structure, @@ -347,6 +369,8 @@ The module :mod:`curses` defines the following functions: bytes object consisting of the prefix ``b'M-'`` followed by the name of the corresponding ASCII character. + Raise a :exc:`ValueError` if *k* is negative. + .. function:: killchar() @@ -372,7 +396,8 @@ The module :mod:`curses` defines the following functions: Set the maximum time in milliseconds that can elapse between press and release events in order for them to be recognized as a click, and return the previous - interval value. The default value is 200 milliseconds, or one fifth of a second. + interval value. The default value is 166 milliseconds, or one sixth of a second. + Use a negative *interval* to obtain the interval value without changing it. .. function:: mousemask(mousemask) @@ -380,7 +405,7 @@ The module :mod:`curses` defines the following functions: Set the mouse events to be reported, and return a tuple ``(availmask, oldmask)``. *availmask* indicates which of the specified mouse events can be reported; on complete failure it returns ``0``. *oldmask* is the previous value of - the given window's mouse event mask. If this function is never called, no mouse + the mouse event mask. If this function is never called, no mouse events are ever reported. @@ -402,7 +427,7 @@ The module :mod:`curses` defines the following functions: methods of a pad require 6 arguments to specify the part of the pad to be displayed and the location on the screen to be used for the display. The arguments are *pminrow*, *pmincol*, *sminrow*, *smincol*, *smaxrow*, *smaxcol*; the *p* - arguments refer to the upper left corner of the pad region to be displayed and + arguments refer to the upper-left corner of the pad region to be displayed and the *s* arguments define a clipping box on the screen within which the pad region is to be displayed. @@ -417,12 +442,14 @@ The module :mod:`curses` defines the following functions: right corner of the screen. -.. function:: nl() +.. function:: nl(flag=True) Enter newline mode. This mode translates the return key into newline on input, and translates newline into return and line-feed on output. Newline mode is initially on. + If *flag* is ``False``, the effect is the same as calling :func:`nonl`. + .. function:: nocbreak() @@ -475,6 +502,8 @@ The module :mod:`curses` defines the following functions: terminfo capability for the current terminal. Note that the output of :func:`putp` always goes to standard output. + :func:`setupterm` (or :func:`initscr`) must be called first. + .. function:: qiflush([flag]) @@ -571,6 +600,10 @@ The module :mod:`curses` defines the following functions: file descriptor to which any initialization sequences will be sent; if not supplied or ``-1``, the file descriptor for ``sys.stdout`` will be used. + Raise a :exc:`curses.error` if the terminal could not be found or its + terminfo database entry could not be read. If the terminal has already + been initialized, this function has no effect. + .. function:: start_color() @@ -579,7 +612,7 @@ The module :mod:`curses` defines the following functions: after :func:`initscr`. :func:`start_color` initializes eight basic colors (black, red, green, yellow, - blue, magenta, cyan, and white), and two global variables in the :mod:`curses` + blue, magenta, cyan, and white), and two global variables in the :mod:`!curses` module, :const:`COLORS` and :const:`COLOR_PAIRS`, containing the maximum number of colors and color-pairs the terminal can support. It also restores the colors on the terminal to the values they had when the terminal was just turned on. @@ -605,6 +638,8 @@ The module :mod:`curses` defines the following functions: Boolean capability, or ``0`` if it is canceled or absent from the terminal description. + :func:`setupterm` (or :func:`initscr`) must be called first. + .. function:: tigetnum(capname) @@ -613,6 +648,8 @@ The module :mod:`curses` defines the following functions: numeric capability, or ``-1`` if it is canceled or absent from the terminal description. + :func:`setupterm` (or :func:`initscr`) must be called first. + .. function:: tigetstr(capname) @@ -621,13 +658,17 @@ The module :mod:`curses` defines the following functions: is not a terminfo "string capability", or is canceled or absent from the terminal description. + :func:`setupterm` (or :func:`initscr`) must be called first. + .. function:: tparm(str[, ...]) Instantiate the bytes object *str* with the supplied parameters, where *str* should - be a parameterized string obtained from the terminfo database. E.g. + be a parameterized string obtained from the terminfo database. For example, ``tparm(tigetstr("cup"), 5, 3)`` could result in ``b'\033[6;4H'``, the exact - result depending on terminal type. + result depending on terminal type. Up to nine integer parameters may be supplied. + + :func:`setupterm` (or :func:`initscr`) must be called first. .. function:: typeahead(fd) @@ -713,11 +754,13 @@ The module :mod:`curses` defines the following functions: .. _curses-window-objects: -Window Objects +Window objects -------------- -Window objects, as returned by :func:`initscr` and :func:`newwin` above, have -the following methods and attributes: +.. class:: window + + Window objects, as returned by :func:`initscr` and :func:`newwin` above, have + the following methods and attributes: .. method:: window.addch(ch[, attr]) @@ -730,7 +773,7 @@ the following methods and attributes: .. note:: Writing outside the window, subwindow, or pad raises a :exc:`curses.error`. - Attempting to write to the lower right corner of a window, subwindow, + Attempting to write to the lower-right corner of a window, subwindow, or pad will cause an exception to be raised after the character is printed. @@ -751,15 +794,14 @@ the following methods and attributes: .. note:: * Writing outside the window, subwindow, or pad raises :exc:`curses.error`. - Attempting to write to the lower right corner of a window, subwindow, + Attempting to write to the lower-right corner of a window, subwindow, or pad will cause an exception to be raised after the string is printed. - * A `bug in ncurses `_, the backend - for this Python module, can cause SegFaults when resizing windows. This - is fixed in ncurses-6.1-20190511. If you are stuck with an earlier - ncurses, you can avoid triggering this if you do not call :func:`addstr` - with a *str* that has embedded newlines. Instead, call :func:`addstr` - separately for each line. + * A bug in ncurses, the backend for this Python module, could cause + segfaults when resizing windows. This was fixed in ncurses-6.1-20190511. + If you are stuck with an earlier ncurses, you can avoid triggering it by + not calling :meth:`!addstr` with a *str* that has embedded newlines; + instead, call :meth:`!addstr` separately for each line. .. method:: window.attroff(attr) @@ -770,7 +812,7 @@ the following methods and attributes: .. method:: window.attron(attr) - Add attribute *attr* from the "background" set applied to all writes to the + Add attribute *attr* to the "background" set applied to all writes to the current window. @@ -886,7 +928,8 @@ the following methods and attributes: .. method:: window.delch([y, x]) - Delete any character at ``(y, x)``. + Delete the character under the cursor, or at ``(y, x)`` if specified. All + characters to the right on the same line are shifted one position left. .. method:: window.deleteln() @@ -988,7 +1031,8 @@ the following methods and attributes: window.getstr(y, x, n) Read a bytes object from the user, with primitive line editing capacity. - The maximum value for *n* is 2047. + At most *n* characters are read; + *n* defaults to and cannot exceed 2047. .. versionchanged:: 3.14 The maximum value for *n* was increased from 1023 to 2047. @@ -1000,11 +1044,12 @@ the following methods and attributes: upper-left corner. -.. method:: window.hline(ch, n) - window.hline(y, x, ch, n) +.. method:: window.hline(ch, n[, attr]) + window.hline(y, x, ch, n[, attr]) Display a horizontal line starting at ``(y, x)`` with length *n* consisting of - the character *ch*. + the character *ch* with attributes *attr*. The line stops at the right edge + of the window if fewer than *n* cells are available. .. method:: window.idcok(flag) @@ -1017,8 +1062,8 @@ the following methods and attributes: .. method:: window.idlok(flag) - If *flag* is ``True``, :mod:`curses` will try and use hardware line - editing facilities. Otherwise, line insertion/deletion are disabled. + If *flag* is ``True``, :mod:`!curses` will try to use hardware line + editing facilities. Otherwise, curses will not use them. .. method:: window.immedok(flag) @@ -1038,8 +1083,10 @@ the following methods and attributes: .. method:: window.insch(ch[, attr]) window.insch(y, x, ch[, attr]) - Paint character *ch* at ``(y, x)`` with attributes *attr*, moving the line from - position *x* right by one character. + Insert character *ch* with attributes *attr* before the character under the + cursor, or at ``(y, x)`` if specified. All characters to the right of the + cursor are shifted one position right, with the rightmost character on the + line being lost. The cursor position does not change. .. method:: window.insdelln(nlines) @@ -1080,7 +1127,8 @@ the following methods and attributes: window.instr(y, x[, n]) Return a bytes object of characters, extracted from the window starting at the - current cursor position, or at *y*, *x* if specified. Attributes are stripped + current cursor position, or at *y*, *x* if specified, and stopping at the end + of the line. Attributes and color information are stripped from the characters. If *n* is specified, :meth:`instr` returns a string at most *n* characters long (exclusive of the trailing NUL). The maximum value for *n* is 2047. @@ -1105,15 +1153,14 @@ the following methods and attributes: .. method:: window.keypad(flag) If *flag* is ``True``, escape sequences generated by some keys (keypad, function keys) - will be interpreted by :mod:`curses`. If *flag* is ``False``, escape sequences will be + will be interpreted by :mod:`!curses`. If *flag* is ``False``, escape sequences will be left as is in the input stream. .. method:: window.leaveok(flag) If *flag* is ``True``, cursor is left where it is on update, instead of being at "cursor - position." This reduces cursor movement where possible. If possible the cursor - will be made invisible. + position." This reduces cursor movement where possible. If *flag* is ``False``, cursor will always be at "cursor position" after an update. @@ -1134,6 +1181,9 @@ the following methods and attributes: Move the window so its upper-left corner is at ``(new_y, new_x)``. + Moving the window so that any part of it would be off the screen is an error: + the window is not moved and :exc:`curses.error` is raised. + .. method:: window.nodelay(flag) @@ -1149,11 +1199,16 @@ the following methods and attributes: .. method:: window.noutrefresh() + window.noutrefresh(pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol) Mark for refresh but wait. This function updates the data structure representing the desired state of the window, but does not force an update of the physical screen. To accomplish that, call :func:`doupdate`. + The 6 arguments can only be specified, and are then required, when the window + is a pad created with :func:`newpad`; they have the same meaning as for + :meth:`refresh`. + .. method:: window.overlay(destwin[, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol]) @@ -1204,12 +1259,12 @@ the following methods and attributes: Update the display immediately (sync actual screen with previous drawing/deleting methods). - The 6 optional arguments can only be specified when the window is a pad created - with :func:`newpad`. The additional parameters are needed to indicate what part - of the pad and screen are involved. *pminrow* and *pmincol* specify the upper - left-hand corner of the rectangle to be displayed in the pad. *sminrow*, + The 6 arguments can only be specified, and are then required, when the window + is a pad created with :func:`newpad`. The additional parameters are needed to indicate what part + of the pad and screen are involved. *pminrow* and *pmincol* specify the + upper-left corner of the rectangle to be displayed in the pad. *sminrow*, *smincol*, *smaxrow*, and *smaxcol* specify the edges of the rectangle to be - displayed on the screen. The lower right-hand corner of the rectangle to be + displayed on the screen. The lower-right corner of the rectangle to be displayed in the pad is calculated from the screen coordinates, since the rectangles must be the same size. Both rectangles must be entirely contained within their respective structures. Negative values of *pminrow*, *pmincol*, @@ -1226,7 +1281,9 @@ the following methods and attributes: .. method:: window.scroll([lines=1]) - Scroll the screen or scrolling region upward by *lines* lines. + Scroll the screen or scrolling region. Scroll upward by *lines* lines if + *lines* is positive, or downward if it is negative. Scrolling has no effect + unless it has been enabled for the window with :meth:`scrollok`. .. method:: window.scrollok(flag) @@ -1259,15 +1316,17 @@ the following methods and attributes: .. method:: window.subpad(begin_y, begin_x) window.subpad(nlines, ncols, begin_y, begin_x) - Return a sub-window, whose upper-left corner is at ``(begin_y, begin_x)``, and - whose width/height is *ncols*/*nlines*. + Return a sub-pad, whose upper-left corner is at ``(begin_y, begin_x)``, and + whose width/height is *ncols*/*nlines*. The coordinates are relative to the + parent pad (unlike :meth:`subwin`, which uses screen coordinates). This + method is only available for pads created with :func:`newpad`. .. method:: window.subwin(begin_y, begin_x) window.subwin(nlines, ncols, begin_y, begin_x) - Return a sub-window, whose upper-left corner is at ``(begin_y, begin_x)``, and - whose width/height is *ncols*/*nlines*. + Return a sub-window, whose upper-left corner is at the screen-relative + coordinates ``(begin_y, begin_x)``, and whose width/height is *ncols*/*nlines*. By default, the sub-window will extend from the specified position to the lower right corner of the window. @@ -1331,7 +1390,7 @@ the following methods and attributes: Constants --------- -The :mod:`curses` module defines the following data members: +The :mod:`!curses` module defines the following data members: .. data:: ERR @@ -1347,7 +1406,6 @@ The :mod:`curses` module defines the following data members: .. data:: version -.. data:: __version__ A bytes object representing the current version of the module. @@ -1375,14 +1433,14 @@ The :mod:`curses` module defines the following data members: .. data:: COLS - The width of the screen, i.e., the number of columns. + The width of the screen, that is, the number of columns. It is defined only after the call to :func:`initscr`. Updated by :func:`update_lines_cols`, :func:`resizeterm` and :func:`resize_term`. .. data:: LINES - The height of the screen, i.e., the number of lines. + The height of the screen, that is, the number of lines. It is defined only after the call to :func:`initscr`. Updated by :func:`update_lines_cols`, :func:`resizeterm` and :func:`resize_term`. @@ -1685,7 +1743,7 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | ACS code | Meaning | +========================+==========================================+ -| .. data:: ACS_BBSS | alternate name for upper right corner | +| .. data:: ACS_BBSS | alternate name for upper-right corner | +------------------------+------------------------------------------+ | .. data:: ACS_BLOCK | solid square block | +------------------------+------------------------------------------+ @@ -1693,7 +1751,7 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | .. data:: ACS_BSBS | alternate name for horizontal line | +------------------------+------------------------------------------+ -| .. data:: ACS_BSSB | alternate name for upper left corner | +| .. data:: ACS_BSSB | alternate name for upper-left corner | +------------------------+------------------------------------------+ | .. data:: ACS_BSSS | alternate name for top tee | +------------------------+------------------------------------------+ @@ -1719,9 +1777,9 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | .. data:: ACS_LEQUAL | less-than-or-equal-to | +------------------------+------------------------------------------+ -| .. data:: ACS_LLCORNER | lower left-hand corner | +| .. data:: ACS_LLCORNER | lower-left corner | +------------------------+------------------------------------------+ -| .. data:: ACS_LRCORNER | lower right-hand corner | +| .. data:: ACS_LRCORNER | lower-right corner | +------------------------+------------------------------------------+ | .. data:: ACS_LTEE | left tee | +------------------------+------------------------------------------+ @@ -1745,13 +1803,13 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | .. data:: ACS_S9 | scan line 9 | +------------------------+------------------------------------------+ -| .. data:: ACS_SBBS | alternate name for lower right corner | +| .. data:: ACS_SBBS | alternate name for lower-right corner | +------------------------+------------------------------------------+ | .. data:: ACS_SBSB | alternate name for vertical line | +------------------------+------------------------------------------+ | .. data:: ACS_SBSS | alternate name for right tee | +------------------------+------------------------------------------+ -| .. data:: ACS_SSBB | alternate name for lower left corner | +| .. data:: ACS_SSBB | alternate name for lower-left corner | +------------------------+------------------------------------------+ | .. data:: ACS_SSBS | alternate name for bottom tee | +------------------------+------------------------------------------+ @@ -1765,9 +1823,9 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | .. data:: ACS_UARROW | up arrow | +------------------------+------------------------------------------+ -| .. data:: ACS_ULCORNER | upper left corner | +| .. data:: ACS_ULCORNER | upper-left corner | +------------------------+------------------------------------------+ -| .. data:: ACS_URCORNER | upper right corner | +| .. data:: ACS_URCORNER | upper-right corner | +------------------------+------------------------------------------+ | .. data:: ACS_VLINE | vertical line | +------------------------+------------------------------------------+ @@ -1791,7 +1849,7 @@ The following table lists mouse button constants used by :meth:`getmouse`: +----------------------------------+---------------------------------------------+ | .. data:: BUTTON_CTRL | Control was down during button state change | +----------------------------------+---------------------------------------------+ -| .. data:: BUTTON_ALT | Control was down during button state change | +| .. data:: BUTTON_ALT | Alt was down during button state change | +----------------------------------+---------------------------------------------+ .. versionchanged:: 3.10 @@ -1821,31 +1879,28 @@ The following table lists the predefined colors: +-------------------------+----------------------------+ -:mod:`curses.textpad` --- Text input widget for curses programs -=============================================================== +:mod:`!curses.textpad` --- Text input widget for curses programs +================================================================ .. module:: curses.textpad :synopsis: Emacs-like input editing in a curses window. -.. moduleauthor:: Eric Raymond -.. sectionauthor:: Eric Raymond - -The :mod:`curses.textpad` module provides a :class:`Textbox` class that handles +The :mod:`!curses.textpad` module provides a :class:`Textbox` class that handles elementary text editing in a curses window, supporting a set of keybindings resembling those of Emacs (thus, also of Netscape Navigator, BBedit 6.x, FrameMaker, and many other programs). The module also provides a rectangle-drawing function useful for framing text boxes or for other purposes. -The module :mod:`curses.textpad` defines the following function: +The module :mod:`!curses.textpad` defines the following function: .. function:: rectangle(win, uly, ulx, lry, lrx) Draw a rectangle. The first argument must be a window object; the remaining arguments are coordinates relative to that window. The second and third - arguments are the y and x coordinates of the upper left hand corner of the + arguments are the y and x coordinates of the upper-left corner of the rectangle to be drawn; the fourth and fifth arguments are the y and x - coordinates of the lower right hand corner. The rectangle will be drawn using + coordinates of the lower-right corner. The rectangle will be drawn using VT100/IBM PC forms characters on terminals that make this possible (including xterm and most other software terminal emulators). Otherwise it will be drawn with ASCII dashes, vertical bars, and plus signs. @@ -1859,32 +1914,36 @@ Textbox objects You can instantiate a :class:`Textbox` object as follows: -.. class:: Textbox(win) +.. class:: Textbox(win, insert_mode=False) Return a textbox widget object. The *win* argument should be a curses :ref:`window ` object in which the textbox is to - be contained. The edit cursor of the textbox is initially located at the - upper left hand corner of the containing window, with coordinates ``(0, 0)``. + be contained. If *insert_mode* is true, the textbox inserts typed + characters, shifting existing text to the right, rather than overwriting it. + The edit cursor of the textbox is initially located at the + upper-left corner of the containing window, with coordinates ``(0, 0)``. The instance's :attr:`stripspaces` flag is initially on. :class:`Textbox` objects have the following methods: - .. method:: edit([validator]) + .. method:: edit(validate=None) This is the entry point you will normally use. It accepts editing keystrokes until one of the termination keystrokes is entered. If - *validator* is supplied, it must be a function. It will be called for + *validate* is supplied, it must be a function. It will be called for each keystroke entered with the keystroke as a parameter; command dispatch - is done on the result. This method returns the window contents as a + is done on the result. If it returns a false value, the keystroke is + ignored. This method returns the window contents as a string; whether blanks in the window are included is affected by the :attr:`stripspaces` attribute. .. method:: do_command(ch) - Process a single command keystroke. Here are the supported special - keystrokes: + Process a single command keystroke. Returns ``1`` to continue editing, + or ``0`` if a termination keystroke was processed. Here are the supported + special keystrokes: +------------------+-------------------------------------------+ | Keystroke | Action | @@ -1907,7 +1966,8 @@ You can instantiate a :class:`Textbox` object as follows: | :kbd:`Control-H` | Delete character backward. | +------------------+-------------------------------------------+ | :kbd:`Control-J` | Terminate if the window is 1 line, | - | | otherwise insert newline. | + | | otherwise move to the start of the next | + | | line. | +------------------+-------------------------------------------+ | :kbd:`Control-K` | If line is blank, delete it, otherwise | | | clear to end of line. | diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 299c8aa399c25c0..ce59d89843e761d 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -4,9 +4,6 @@ .. module:: dataclasses :synopsis: Generate special methods on user-defined classes. -.. moduleauthor:: Eric V. Smith -.. sectionauthor:: Eric V. Smith - **Source code:** :source:`Lib/dataclasses.py` -------------- @@ -103,12 +100,25 @@ Module contents ignored. - *eq*: If true (the default), an :meth:`~object.__eq__` method will be - generated. This method compares the class as if it were a tuple - of its fields, in order. Both instances in the comparison must - be of the identical type. + generated. - If the class already defines :meth:`!__eq__`, this parameter is - ignored. + This method compares the class by comparing each field in order. Both + instances in the comparison must be of the identical type. + + If the class already defines :meth:`!__eq__`, this parameter is ignored. + + .. versionchanged:: 3.13 + The generated ``__eq__`` method now compares each field individually + (for example, ``self.a == other.a and self.b == other.b``), rather than + comparing tuples of fields as in previous versions. + + This change makes the comparison faster but it may alter results in cases + where attributes compare equal by identity but not by value (such as + ``float('nan')``). + + In Python 3.12 and earlier, the comparison was performed by creating + tuples of the fields and comparing them (for example, + ``(self.a, self.b) == (other.a, other.b)``). - *order*: If true (the default is ``False``), :meth:`~object.__lt__`, :meth:`~object.__le__`, :meth:`~object.__gt__`, and :meth:`~object.__ge__` methods will be @@ -161,9 +171,11 @@ Module contents :class:`object`, this means it will fall back to id-based hashing). - *frozen*: If true (the default is ``False``), assigning to fields will - generate an exception. This emulates read-only frozen instances. If - :meth:`~object.__setattr__` or :meth:`~object.__delattr__` is defined in the class, then - :exc:`TypeError` is raised. See the discussion below. + generate an exception. This emulates read-only frozen instances. + See the :ref:`discussion ` below. + + If :meth:`~object.__setattr__` or :meth:`~object.__delattr__` is defined in the class + and *frozen* is true, then :exc:`TypeError` is raised. - *match_args*: If true (the default is ``True``), the :attr:`~object.__match_args__` tuple will be created from the list of @@ -315,7 +327,7 @@ Module contents :func:`!field`, then the class attribute for this field will be replaced by the specified *default* value. If *default* is not provided, then the class attribute will be deleted. The intent is - that after the :func:`@dataclass ` decorator runs, the class + that after the :deco:`dataclass` decorator runs, the class attributes will all contain the default values for the fields, just as if the default value itself were specified. For example, after:: @@ -331,6 +343,10 @@ Module contents :attr:`!C.t` will be ``20``, and the class attributes :attr:`!C.x` and :attr:`!C.y` will not be set. + .. versionchanged:: 3.15 + If *metadata* is ``None``, use an empty :class:`frozendict`, instead + of a :func:`~types.MappingProxyType` of an empty :class:`dict`. + .. class:: Field :class:`!Field` objects describe each defined field. These objects @@ -368,8 +384,8 @@ Module contents Converts the dataclass *obj* to a dict (by using the factory function *dict_factory*). Each dataclass is converted to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts, - lists, and tuples are recursed into. Other objects are copied with - :func:`copy.deepcopy`. + frozendicts, lists, and tuples are recursed into. Other objects are copied + with :func:`copy.deepcopy`. Example of using :func:`!asdict` on nested dataclasses:: @@ -399,8 +415,8 @@ Module contents Converts the dataclass *obj* to a tuple (by using the factory function *tuple_factory*). Each dataclass is converted - to a tuple of its field values. dataclasses, dicts, lists, and - tuples are recursed into. Other objects are copied with + to a tuple of its field values. dataclasses, dicts, frozendicts, lists, + and tuples are recursed into. Other objects are copied with :func:`copy.deepcopy`. Continuing from the previous example:: @@ -415,7 +431,7 @@ Module contents :func:`!astuple` raises :exc:`TypeError` if *obj* is not a dataclass instance. -.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None, decorator=dataclass) +.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None, qualname=None, decorator=dataclass) Creates a new dataclass with name *cls_name*, fields as defined in *fields*, base classes as given in *bases*, and initialized @@ -425,20 +441,23 @@ Module contents :data:`typing.Any` is used for ``type``. The values of *init*, *repr*, *eq*, *order*, *unsafe_hash*, *frozen*, *match_args*, *kw_only*, *slots*, and *weakref_slot* have - the same meaning as they do in :func:`@dataclass `. + the same meaning as they do in :deco:`dataclass`. If *module* is defined, the :attr:`!__module__` attribute of the dataclass is set to that value. By default, it is set to the module name of the caller. + If *qualname* is defined, the :attr:`~type.__qualname__` attribute of the dataclass + is set to that value. By default, it is set to the value passed to *cls_name*. + The *decorator* parameter is a callable that will be used to create the dataclass. It should take the class object as a first argument and the same keyword arguments - as :func:`@dataclass `. By default, the :func:`@dataclass ` + as :deco:`dataclass`. By default, the :deco:`dataclass` function is used. This function is not strictly required, because any Python - mechanism for creating a new class with :attr:`!__annotations__` can - then apply the :func:`@dataclass ` function to convert that class to + mechanism for creating a new class with :attr:`~object.__annotations__` can + then apply the :deco:`dataclass` function to convert that class to a dataclass. This function is provided as a convenience. For example:: @@ -461,6 +480,8 @@ Module contents .. versionadded:: 3.14 Added the *decorator* parameter. + .. versionadded:: next + Added the *qualname* parameter. .. function:: replace(obj, /, **changes) @@ -495,7 +516,8 @@ Module contents .. function:: is_dataclass(obj) Return ``True`` if its parameter is a dataclass (including subclasses of a - dataclass) or an instance of one, otherwise return ``False``. + dataclass, but not including :ref:`generic aliases `) + or an instance of one, otherwise return ``False``. If you need to know if a class is an instance of a dataclass (and not a dataclass itself), then add a further check for ``not @@ -567,7 +589,7 @@ Post-init processing def __post_init__(self): self.c = self.a + self.b -The :meth:`~object.__init__` method generated by :func:`@dataclass ` does not call base +The :meth:`~object.__init__` method generated by :deco:`dataclass` does not call base class :meth:`!__init__` methods. If the base class has an :meth:`!__init__` method that has to be called, it is common to call this method in a :meth:`__post_init__` method:: @@ -597,7 +619,7 @@ parameters to :meth:`!__post_init__`. Also see the warning about how Class variables --------------- -One of the few places where :func:`@dataclass ` actually inspects the type +One of the few places where :deco:`dataclass` actually inspects the type of a field is to determine if a field is a class variable as defined in :pep:`526`. It does this by checking if the type of the field is :data:`typing.ClassVar`. If a field is a ``ClassVar``, it is excluded @@ -610,7 +632,7 @@ module-level :func:`fields` function. Init-only variables ------------------- -Another place where :func:`@dataclass ` inspects a type annotation is to +Another place where :deco:`dataclass` inspects a type annotation is to determine if a field is an init-only variable. It does this by seeing if the type of a field is of type :class:`InitVar`. If a field is an :class:`InitVar`, it is considered a pseudo-field called an init-only @@ -644,7 +666,7 @@ Frozen instances ---------------- It is not possible to create truly immutable Python objects. However, -by passing ``frozen=True`` to the :func:`@dataclass ` decorator you can +by passing ``frozen=True`` to the :deco:`dataclass` decorator you can emulate immutability. In that case, dataclasses will add :meth:`~object.__setattr__` and :meth:`~object.__delattr__` methods to the class. These methods will raise a :exc:`FrozenInstanceError` when invoked. @@ -660,7 +682,7 @@ must use :meth:`!object.__setattr__`. Inheritance ----------- -When the dataclass is being created by the :func:`@dataclass ` decorator, +When the dataclass is being created by the :deco:`dataclass` decorator, it looks through all of the class's base classes in reverse MRO (that is, starting at :class:`object`) and, for each dataclass that it finds, adds the fields from that base class to an ordered mapping of fields. @@ -784,7 +806,7 @@ for :attr:`!x` when creating a class instance will share the same copy of :attr:`!x`. Because dataclasses just use normal Python class creation they also share this behavior. There is no general way for Data Classes to detect this condition. Instead, the -:func:`@dataclass ` decorator will raise a :exc:`ValueError` if it +:deco:`dataclass` decorator will raise a :exc:`ValueError` if it detects an unhashable default parameter. The assumption is that if a value is unhashable, it is mutable. This is a partial solution, but it does protect against many common errors. @@ -818,7 +840,7 @@ default value have the following special behaviors: :meth:`~object.__get__` or :meth:`!__set__` method is called rather than returning or overwriting the descriptor object. -* To determine whether a field contains a default value, :func:`@dataclass ` +* To determine whether a field contains a default value, :deco:`dataclass` will call the descriptor's :meth:`!__get__` method using its class access form: ``descriptor.__get__(obj=None, type=cls)``. If the descriptor returns a value in this case, it will be used as the diff --git a/Doc/library/datetime-inheritance.dot b/Doc/library/datetime-inheritance.dot new file mode 100644 index 000000000000000..3c6b9b4beb7ab1d --- /dev/null +++ b/Doc/library/datetime-inheritance.dot @@ -0,0 +1,31 @@ +// Used to generate datetime-inheritance.svg with Graphviz +// (https://graphviz.org/) for the datetime documentation. + +digraph { + comment="Generated with datetime-inheritance.dot" + graph [ + bgcolor="transparent" + fontnames="svg" + layout="dot" + ranksep=0.5 + nodesep=0.5 + splines=line + ] + node [ + fontname="Courier" + fontsize=14.0 + shape=box + style=rounded + margin="0.15,0.07" + ] + edge [ + arrowhead=none + ] + + object -> tzinfo + object -> timedelta + object -> time + object -> date + tzinfo -> timezone + date -> datetime +} diff --git a/Doc/library/datetime-inheritance.svg b/Doc/library/datetime-inheritance.svg new file mode 100644 index 000000000000000..e6b1cf877a574f2 --- /dev/null +++ b/Doc/library/datetime-inheritance.svg @@ -0,0 +1,84 @@ + + + + + + +datetime class hierarchy + + +object + +object + + + +tzinfo + +tzinfo + + + +object->tzinfo + + + + +timedelta + +timedelta + + + +object->timedelta + + + + +time + +time + + + +object->time + + + + +date + +date + + + +object->date + + + + +timezone + +timezone + + + +tzinfo->timezone + + + + +datetime + +datetime + + + +date->datetime + + + + diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 16ed3215bc2c1aa..f3c4ef9199075c9 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -4,16 +4,10 @@ .. module:: datetime :synopsis: Basic date and time types. -.. moduleauthor:: Tim Peters -.. sectionauthor:: Tim Peters -.. sectionauthor:: A.M. Kuchling - **Source code:** :source:`Lib/datetime.py` -------------- -.. XXX what order should the types be discussed in? - The :mod:`!datetime` module supplies classes for manipulating dates and times. While date and time arithmetic is supported, the focus of the implementation is @@ -38,13 +32,14 @@ on efficient attribute extraction for output formatting and manipulation. Third-party library with expanded time zone and parsing support. Package :pypi:`DateType` - Third-party library that introduces distinct static types to e.g. allow - :term:`static type checkers ` + Third-party library that introduces distinct static types to for example, + allow :term:`static type checkers ` to differentiate between naive and aware datetimes. + .. _datetime-naive-aware: -Aware and Naive Objects +Aware and naive objects ----------------------- Date and time objects may be categorized as "aware" or "naive" depending on @@ -65,7 +60,7 @@ understand and to work with, at the cost of ignoring some aspects of reality. For applications requiring aware objects, :class:`.datetime` and :class:`.time` objects have an optional time zone information attribute, :attr:`!tzinfo`, that -can be set to an instance of a subclass of the abstract :class:`tzinfo` class. +can be set to an instance of a subclass of the abstract :class:`!tzinfo` class. These :class:`tzinfo` objects capture information about the offset from UTC time, the time zone name, and whether daylight saving time is in effect. @@ -77,6 +72,7 @@ detail is up to the application. The rules for time adjustment across the world are more political than rational, change frequently, and there is no standard suitable for every application aside from UTC. + Constants --------- @@ -93,13 +89,15 @@ The :mod:`!datetime` module exports the following constants: The largest year number allowed in a :class:`date` or :class:`.datetime` object. :const:`MAXYEAR` is 9999. + .. data:: UTC Alias for the UTC time zone singleton :attr:`datetime.timezone.utc`. .. versionadded:: 3.11 -Available Types + +Available types --------------- .. class:: date @@ -142,6 +140,7 @@ Available Types time adjustment (for example, to account for time zone and/or daylight saving time). + .. class:: timezone :noindex: @@ -150,19 +149,19 @@ Available Types .. versionadded:: 3.2 + Objects of these types are immutable. -Subclass relationships:: +Subclass relationships: + +.. figure:: datetime-inheritance.svg + :class: invert-in-dark-mode + :align: center + :alt: timedelta, tzinfo, time, and date inherit from object; timezone inherits + from tzinfo; and datetime inherits from date. - object - timedelta - tzinfo - timezone - time - date - datetime -Common Properties +Common properties ^^^^^^^^^^^^^^^^^ The :class:`date`, :class:`.datetime`, :class:`.time`, and :class:`timezone` types @@ -173,7 +172,8 @@ share these common features: dictionary keys. - Objects of these types support efficient pickling via the :mod:`pickle` module. -Determining if an Object is Aware or Naive + +Determining if an object is aware or naive ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Objects of the :class:`date` type are always naive. @@ -197,10 +197,11 @@ Otherwise, ``t`` is naive. The distinction between aware and naive doesn't apply to :class:`timedelta` objects. + .. _datetime-timedelta: -:class:`timedelta` Objects --------------------------- +:class:`!timedelta` objects +--------------------------- A :class:`timedelta` object represents a duration, the difference between two :class:`.datetime` or :class:`date` instances. @@ -229,8 +230,8 @@ A :class:`timedelta` object represents a duration, the difference between two *days*, *seconds* and *microseconds* are "merged" and normalized into those three resulting attributes:: - >>> from datetime import timedelta - >>> delta = timedelta( + >>> import datetime as dt + >>> delta = dt.timedelta( ... days=50, ... seconds=27, ... microseconds=10, @@ -243,6 +244,12 @@ A :class:`timedelta` object represents a duration, the difference between two >>> delta datetime.timedelta(days=64, seconds=29156, microseconds=10) + .. tip:: + ``import datetime as dt`` instead of ``import datetime`` or + ``from datetime import datetime`` to avoid confusion between the module + and the class. See `How I Import Python’s datetime Module + `__. + If any argument is a float and there are fractional microseconds, the fractional microseconds left over from all arguments are combined and their sum is rounded to the nearest microsecond using @@ -256,8 +263,8 @@ A :class:`timedelta` object represents a duration, the difference between two Note that normalization of negative values may be surprising at first. For example:: - >>> from datetime import timedelta - >>> d = timedelta(microseconds=-1) + >>> import datetime as dt + >>> d = dt.timedelta(microseconds=-1) >>> (d.days, d.seconds, d.microseconds) (-1, 86399, 999999) @@ -296,6 +303,7 @@ Class attributes: The smallest possible difference between non-equal :class:`timedelta` objects, ``timedelta(microseconds=1)``. + Note that, because of normalization, ``timedelta.max`` is greater than ``-timedelta.min``. ``-timedelta.max`` is not representable as a :class:`timedelta` object. @@ -319,13 +327,14 @@ Instance attributes (read-only): .. doctest:: - >>> from datetime import timedelta - >>> duration = timedelta(seconds=11235813) + >>> import datetime as dt + >>> duration = dt.timedelta(seconds=11235813) >>> duration.days, duration.seconds (130, 3813) >>> duration.total_seconds() 11235813.0 + .. attribute:: timedelta.microseconds Between 0 and 999,999 inclusive. @@ -333,8 +342,6 @@ Instance attributes (read-only): Supported operations: -.. XXX this table is too wide! - +--------------------------------+-----------------------------------------------+ | Operation | Result | +================================+===============================================+ @@ -396,7 +403,6 @@ Supported operations: | | call with canonical attribute values. | +--------------------------------+-----------------------------------------------+ - Notes: (1) @@ -432,9 +438,9 @@ objects (see below). .. versionchanged:: 3.2 Floor division and true division of a :class:`timedelta` object by another - :class:`timedelta` object are now supported, as are remainder operations and + :class:`!timedelta` object are now supported, as are remainder operations and the :func:`divmod` function. True division and multiplication of a - :class:`timedelta` object by a :class:`float` object are now supported. + :class:`!timedelta` object by a :class:`float` object are now supported. :class:`timedelta` objects support equality and order comparisons. @@ -447,23 +453,24 @@ Instance methods: Return the total number of seconds contained in the duration. Equivalent to ``td / timedelta(seconds=1)``. For interval units other than seconds, use the - division form directly (e.g. ``td / timedelta(microseconds=1)``). + division form directly (for example, ``td / timedelta(microseconds=1)``). Note that for very large time intervals (greater than 270 years on most platforms) this method will lose microsecond accuracy. .. versionadded:: 3.2 -Examples of usage: :class:`timedelta` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Examples of usage: :class:`!timedelta` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ An additional example of normalization:: >>> # Components of another_year add up to exactly 365 days - >>> from datetime import timedelta - >>> year = timedelta(days=365) - >>> another_year = timedelta(weeks=40, days=84, hours=23, - ... minutes=50, seconds=600) + >>> import datetime as dt + >>> year = dt.timedelta(days=365) + >>> another_year = dt.timedelta(weeks=40, days=84, hours=23, + ... minutes=50, seconds=600) >>> year == another_year True >>> year.total_seconds() @@ -471,8 +478,8 @@ An additional example of normalization:: Examples of :class:`timedelta` arithmetic:: - >>> from datetime import timedelta - >>> year = timedelta(days=365) + >>> import datetime as dt + >>> year = dt.timedelta(days=365) >>> ten_years = 10 * year >>> ten_years datetime.timedelta(days=3650) @@ -485,10 +492,11 @@ Examples of :class:`timedelta` arithmetic:: >>> three_years, three_years.days // 365 (datetime.timedelta(days=1095), 3) + .. _datetime-date: -:class:`date` Objects ---------------------- +:class:`!date` objects +---------------------- A :class:`date` object represents a date (year, month and day) in an idealized calendar, the current Gregorian calendar indefinitely extended in both @@ -517,9 +525,10 @@ Other constructors, all class methods: This is equivalent to ``date.fromtimestamp(time.time())``. + .. classmethod:: date.fromtimestamp(timestamp) - Return the local date corresponding to the POSIX timestamp, such as is + Return the local date corresponding to the POSIX *timestamp*, such as is returned by :func:`time.time`. This may raise :exc:`OverflowError`, if the timestamp is out @@ -535,10 +544,13 @@ Other constructors, all class methods: :c:func:`localtime` function. Raise :exc:`OSError` instead of :exc:`ValueError` on :c:func:`localtime` failure. + .. versionchanged:: 3.15 + Accepts any real number as *timestamp*, not only integer or float. + .. classmethod:: date.fromordinal(ordinal) - Return the date corresponding to the proleptic Gregorian ordinal, where + Return the date corresponding to the proleptic Gregorian *ordinal*, where January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 <= ordinal <= @@ -559,25 +571,27 @@ Other constructors, all class methods: Examples:: - >>> from datetime import date - >>> date.fromisoformat('2019-12-04') + >>> import datetime as dt + >>> dt.date.fromisoformat('2019-12-04') datetime.date(2019, 12, 4) - >>> date.fromisoformat('20191204') + >>> dt.date.fromisoformat('20191204') datetime.date(2019, 12, 4) - >>> date.fromisoformat('2021-W01-1') + >>> dt.date.fromisoformat('2021-W01-1') datetime.date(2021, 1, 4) .. versionadded:: 3.7 .. versionchanged:: 3.11 Previously, this method only supported the format ``YYYY-MM-DD``. + .. classmethod:: date.fromisocalendar(year, week, day) Return a :class:`date` corresponding to the ISO calendar date specified by - year, week and day. This is the inverse of the function :meth:`date.isocalendar`. + *year*, *week* and *day*. This is the inverse of the function :meth:`date.isocalendar`. .. versionadded:: 3.8 + .. classmethod:: date.strptime(date_string, format) Return a :class:`.date` corresponding to *date_string*, parsed according to @@ -592,20 +606,19 @@ Other constructors, all class methods: .. note:: - If *format* specifies a day of month without a year a - :exc:`DeprecationWarning` is emitted. This is to avoid a quadrennial + If *format* specifies a day of month (``%d``) without a year, + :exc:`ValueError` is raised. This is to avoid a quadrennial leap year bug in code seeking to parse only a month and day as the default year used in absence of one in the format is not a leap year. - Such *format* values may raise an error as of Python 3.15. The - workaround is to always include a year in your *format*. If parsing + The workaround is to always include a year in your *format*. If parsing *date_string* values that do not have a year, explicitly add a year that is a leap year before parsing: .. doctest:: - >>> from datetime import date + >>> import datetime as dt >>> date_string = "02/29" - >>> when = date.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. + >>> when = dt.date.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. >>> when.strftime("%B %d") # doctest: +SKIP 'February 29' @@ -697,7 +710,7 @@ Notes: In other words, ``date1 < date2`` if and only if ``date1.toordinal() < date2.toordinal()``. - Order comparison between a :class:`!date` object that is not also a + Order comparison between a :class:`date` object that is not also a :class:`.datetime` instance and a :class:`!datetime` object raises :exc:`TypeError`. @@ -720,8 +733,8 @@ Instance methods: Example:: - >>> from datetime import date - >>> d = date(2002, 12, 31) + >>> import datetime as dt + >>> d = dt.date(2002, 12, 31) >>> d.replace(day=26) datetime.date(2002, 12, 26) @@ -779,23 +792,25 @@ Instance methods: For example, 2004 begins on a Thursday, so the first week of ISO year 2004 begins on Monday, 29 Dec 2003 and ends on Sunday, 4 Jan 2004:: - >>> from datetime import date - >>> date(2003, 12, 29).isocalendar() + >>> import datetime as dt + >>> dt.date(2003, 12, 29).isocalendar() datetime.IsoCalendarDate(year=2004, week=1, weekday=1) - >>> date(2004, 1, 4).isocalendar() + >>> dt.date(2004, 1, 4).isocalendar() datetime.IsoCalendarDate(year=2004, week=1, weekday=7) .. versionchanged:: 3.9 Result changed from a tuple to a :term:`named tuple`. + .. method:: date.isoformat() Return a string representing the date in ISO 8601 format, ``YYYY-MM-DD``:: - >>> from datetime import date - >>> date(2002, 12, 4).isoformat() + >>> import datetime as dt + >>> dt.date(2002, 12, 4).isoformat() '2002-12-04' + .. method:: date.__str__() For a date ``d``, ``str(d)`` is equivalent to ``d.isoformat()``. @@ -805,8 +820,8 @@ Instance methods: Return a string representing the date:: - >>> from datetime import date - >>> date(2002, 12, 4).ctime() + >>> import datetime as dt + >>> dt.date(2002, 12, 4).ctime() 'Wed Dec 4 00:00:00 2002' ``d.ctime()`` is equivalent to:: @@ -832,19 +847,20 @@ Instance methods: literals ` and when using :meth:`str.format`. See also :ref:`strftime-strptime-behavior` and :meth:`date.isoformat`. -Examples of Usage: :class:`date` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Examples of usage: :class:`!date` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Example of counting days to an event:: >>> import time - >>> from datetime import date - >>> today = date.today() + >>> import datetime as dt + >>> today = dt.date.today() >>> today datetime.date(2007, 12, 5) - >>> today == date.fromtimestamp(time.time()) + >>> today == dt.date.fromtimestamp(time.time()) True - >>> my_birthday = date(today.year, 6, 24) + >>> my_birthday = dt.date(today.year, 6, 24) >>> if my_birthday < today: ... my_birthday = my_birthday.replace(year=today.year + 1) ... @@ -858,8 +874,8 @@ More examples of working with :class:`date`: .. doctest:: - >>> from datetime import date - >>> d = date.fromordinal(730920) # 730920th day after 1. 1. 0001 + >>> import datetime as dt + >>> d = dt.date.fromordinal(730920) # 730920th day after 1. 1. 0001 >>> d datetime.date(2002, 3, 11) @@ -875,7 +891,7 @@ More examples of working with :class:`date`: >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month") 'The day is 11, the month is March.' - >>> # Methods for to extracting 'components' under different calendars + >>> # Methods for extracting 'components' under different calendars >>> t = d.timetuple() >>> for i in t: # doctest: +SKIP ... print(i) @@ -902,7 +918,7 @@ More examples of working with :class:`date`: .. _datetime-datetime: -:class:`.datetime` Objects +:class:`!datetime` objects -------------------------- A :class:`.datetime` object is a single object containing all the information @@ -910,7 +926,7 @@ from a :class:`date` object and a :class:`.time` object. Like a :class:`date` object, :class:`.datetime` assumes the current Gregorian calendar extended in both directions; like a :class:`.time` object, -:class:`.datetime` assumes there are exactly 3600\*24 seconds in every day. +:class:`!datetime` assumes there are exactly 3600\*24 seconds in every day. Constructor: @@ -934,6 +950,7 @@ Constructor: .. versionchanged:: 3.6 Added the *fold* parameter. + Other constructors, all class methods: .. classmethod:: datetime.today() @@ -949,6 +966,7 @@ Other constructors, all class methods: This method is functionally equivalent to :meth:`now`, but without a ``tz`` parameter. + .. classmethod:: datetime.now(tz=None) Return the current local date and time. @@ -969,6 +987,7 @@ Other constructors, all class methods: Subsequent calls to :meth:`!datetime.now` may return the same instant depending on the precision of the underlying clock. + .. classmethod:: datetime.utcnow() Return the current UTC date and time, with :attr:`.tzinfo` ``None``. @@ -1020,6 +1039,10 @@ Other constructors, all class methods: .. versionchanged:: 3.6 :meth:`fromtimestamp` may return instances with :attr:`.fold` set to 1. + .. versionchanged:: 3.15 + Accepts any real number as *timestamp*, not only integer or float. + + .. classmethod:: datetime.utcfromtimestamp(timestamp) Return the UTC :class:`.datetime` corresponding to the POSIX timestamp, with @@ -1056,6 +1079,9 @@ Other constructors, all class methods: :c:func:`gmtime` function. Raise :exc:`OSError` instead of :exc:`ValueError` on :c:func:`gmtime` failure. + .. versionchanged:: 3.15 + Accepts any real number as *timestamp*, not only integer or float. + .. deprecated:: 3.12 Use :meth:`datetime.fromtimestamp` with :const:`UTC` instead. @@ -1076,7 +1102,7 @@ Other constructors, all class methods: are equal to the given :class:`.time` object's. If the *tzinfo* argument is provided, its value is used to set the :attr:`.tzinfo` attribute of the result, otherwise the :attr:`~.time.tzinfo` attribute of the *time* argument - is used. If the *date* argument is a :class:`.datetime` object, its time components + is used. If the *date* argument is a :class:`!datetime` object, its time components and :attr:`.tzinfo` attributes are ignored. For any :class:`.datetime` object ``d``, @@ -1102,24 +1128,24 @@ Other constructors, all class methods: Examples:: - >>> from datetime import datetime - >>> datetime.fromisoformat('2011-11-04') + >>> import datetime as dt + >>> dt.datetime.fromisoformat('2011-11-04') datetime.datetime(2011, 11, 4, 0, 0) - >>> datetime.fromisoformat('20111104') + >>> dt.datetime.fromisoformat('20111104') datetime.datetime(2011, 11, 4, 0, 0) - >>> datetime.fromisoformat('2011-11-04T00:05:23') + >>> dt.datetime.fromisoformat('2011-11-04T00:05:23') datetime.datetime(2011, 11, 4, 0, 5, 23) - >>> datetime.fromisoformat('2011-11-04T00:05:23Z') + >>> dt.datetime.fromisoformat('2011-11-04T00:05:23Z') datetime.datetime(2011, 11, 4, 0, 5, 23, tzinfo=datetime.timezone.utc) - >>> datetime.fromisoformat('20111104T000523') + >>> dt.datetime.fromisoformat('20111104T000523') datetime.datetime(2011, 11, 4, 0, 5, 23) - >>> datetime.fromisoformat('2011-W01-2T00:05:23.283') + >>> dt.datetime.fromisoformat('2011-W01-2T00:05:23.283') datetime.datetime(2011, 1, 4, 0, 5, 23, 283000) - >>> datetime.fromisoformat('2011-11-04 00:05:23.283') + >>> dt.datetime.fromisoformat('2011-11-04 00:05:23.283') datetime.datetime(2011, 11, 4, 0, 5, 23, 283000) - >>> datetime.fromisoformat('2011-11-04 00:05:23.283+00:00') + >>> dt.datetime.fromisoformat('2011-11-04 00:05:23.283+00:00') datetime.datetime(2011, 11, 4, 0, 5, 23, 283000, tzinfo=datetime.timezone.utc) - >>> datetime.fromisoformat('2011-11-04T00:05:23+04:00') # doctest: +NORMALIZE_WHITESPACE + >>> dt.datetime.fromisoformat('2011-11-04T00:05:23+04:00') # doctest: +NORMALIZE_WHITESPACE datetime.datetime(2011, 11, 4, 0, 5, 23, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) @@ -1132,12 +1158,13 @@ Other constructors, all class methods: .. classmethod:: datetime.fromisocalendar(year, week, day) Return a :class:`.datetime` corresponding to the ISO calendar date specified - by year, week and day. The non-date components of the datetime are populated + by *year*, *week* and *day*. The non-date components of the datetime are populated with their normal default values. This is the inverse of the function :meth:`datetime.isocalendar`. .. versionadded:: 3.8 + .. classmethod:: datetime.strptime(date_string, format) Return a :class:`.datetime` corresponding to *date_string*, parsed according to @@ -1152,22 +1179,21 @@ Other constructors, all class methods: time tuple. See also :ref:`strftime-strptime-behavior` and :meth:`datetime.fromisoformat`. - .. versionchanged:: 3.13 + .. versionchanged:: 3.15 - If *format* specifies a day of month without a year a - :exc:`DeprecationWarning` is now emitted. This is to avoid a quadrennial + If *format* specifies a day of month (``%d``) without a year, + :exc:`ValueError` is raised. This is to avoid a quadrennial leap year bug in code seeking to parse only a month and day as the default year used in absence of one in the format is not a leap year. - Such *format* values may raise an error as of Python 3.15. The - workaround is to always include a year in your *format*. If parsing + The workaround is to always include a year in your *format*. If parsing *date_string* values that do not have a year, explicitly add a year that is a leap year before parsing: .. doctest:: - >>> from datetime import datetime + >>> import datetime as dt >>> date_string = "02/29" - >>> when = datetime.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. + >>> when = dt.datetime.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. >>> when.strftime("%B %d") # doctest: +SKIP 'February 29' @@ -1245,6 +1271,7 @@ Instance attributes (read-only): .. versionadded:: 3.6 + Supported operations: +---------------------------------------+--------------------------------+ @@ -1280,7 +1307,7 @@ Supported operations: datetime, and no time zone adjustments are done even if the input is aware. (3) - Subtraction of a :class:`.datetime` from a :class:`.datetime` is defined only if + Subtraction of a :class:`.datetime` from a :class:`!datetime` is defined only if both operands are naive, or if both are aware. If one is aware and the other is naive, :exc:`TypeError` is raised. @@ -1298,7 +1325,7 @@ Supported operations: :class:`.datetime` objects are equal if they represent the same date and time, taking into account the time zone. - Naive and aware :class:`!datetime` objects are never equal. + Naive and aware :class:`.datetime` objects are never equal. If both comparands are aware, and have the same :attr:`!tzinfo` attribute, the :attr:`!tzinfo` and :attr:`~.datetime.fold` attributes are ignored and @@ -1306,7 +1333,7 @@ Supported operations: If both comparands are aware and have different :attr:`~.datetime.tzinfo` attributes, the comparison acts as comparands were first converted to UTC datetimes except that the implementation never overflows. - :class:`!datetime` instances in a repeated interval are never equal to + :class:`.datetime` instances in a repeated interval are never equal to :class:`!datetime` instances in other time zone. (5) @@ -1335,6 +1362,7 @@ Supported operations: The default behavior can be changed by overriding the special comparison methods in subclasses. + Instance methods: .. method:: datetime.date() @@ -1490,11 +1518,13 @@ Instance methods: ``datetime.replace(tzinfo=timezone.utc)`` to make it aware, at which point you can use :meth:`.datetime.timetuple`. + .. method:: datetime.toordinal() Return the proleptic Gregorian ordinal of the date. The same as ``self.date().toordinal()``. + .. method:: datetime.timestamp() Return POSIX timestamp corresponding to the :class:`.datetime` @@ -1503,7 +1533,7 @@ Instance methods: Naive :class:`.datetime` instances are assumed to represent local time and this method relies on platform C functions to perform - the conversion. Since :class:`.datetime` supports a wider range of + the conversion. Since :class:`!datetime` supports a wider range of values than the platform C functions on many platforms, this method may raise :exc:`OverflowError` or :exc:`OSError` for times far in the past or far in the future. @@ -1513,16 +1543,6 @@ Instance methods: (dt - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() - .. versionadded:: 3.3 - - .. versionchanged:: 3.6 - The :meth:`timestamp` method uses the :attr:`.fold` attribute to - disambiguate the times during a repeated interval. - - .. versionchanged:: 3.6 - This method no longer relies on the platform C :c:func:`mktime` - function to perform conversions. - .. note:: There is no method to obtain the POSIX timestamp directly from a @@ -1537,6 +1557,17 @@ Instance methods: timestamp = (dt - datetime(1970, 1, 1)) / timedelta(seconds=1) + .. versionadded:: 3.3 + + .. versionchanged:: 3.6 + The :meth:`timestamp` method uses the :attr:`.fold` attribute to + disambiguate the times during a repeated interval. + + .. versionchanged:: 3.6 + This method no longer relies on the platform C :c:func:`mktime` + function to perform conversions. + + .. method:: datetime.weekday() Return the day of the week as an integer, where Monday is 0 and Sunday is 6. @@ -1572,24 +1603,24 @@ Instance methods: Examples:: - >>> from datetime import datetime, timezone - >>> datetime(2019, 5, 18, 15, 17, 8, 132263).isoformat() + >>> import datetime as dt + >>> dt.datetime(2019, 5, 18, 15, 17, 8, 132263).isoformat() '2019-05-18T15:17:08.132263' - >>> datetime(2019, 5, 18, 15, 17, tzinfo=timezone.utc).isoformat() + >>> dt.datetime(2019, 5, 18, 15, 17, tzinfo=dt.timezone.utc).isoformat() '2019-05-18T15:17:00+00:00' The optional argument *sep* (default ``'T'``) is a one-character separator, placed between the date and time portions of the result. For example:: - >>> from datetime import tzinfo, timedelta, datetime - >>> class TZ(tzinfo): + >>> import datetime as dt + >>> class TZ(dt.tzinfo): ... """A time zone with an arbitrary, constant -06:39 offset.""" - ... def utcoffset(self, dt): - ... return timedelta(hours=-6, minutes=-39) + ... def utcoffset(self, when): + ... return dt.timedelta(hours=-6, minutes=-39) ... - >>> datetime(2002, 12, 25, tzinfo=TZ()).isoformat(' ') + >>> dt.datetime(2002, 12, 25, tzinfo=TZ()).isoformat(' ') '2002-12-25 00:00:00-06:39' - >>> datetime(2009, 11, 27, microsecond=100, tzinfo=TZ()).isoformat() + >>> dt.datetime(2009, 11, 27, microsecond=100, tzinfo=TZ()).isoformat() '2009-11-27T00:00:00.000100-06:39' The optional argument *timespec* specifies the number of additional @@ -1613,11 +1644,11 @@ Instance methods: :exc:`ValueError` will be raised on an invalid *timespec* argument:: - >>> from datetime import datetime - >>> datetime.now().isoformat(timespec='minutes') # doctest: +SKIP + >>> import datetime as dt + >>> dt.datetime.now().isoformat(timespec='minutes') # doctest: +SKIP '2002-12-25T00:00' - >>> dt = datetime(2015, 1, 1, 12, 30, 59, 0) - >>> dt.isoformat(timespec='microseconds') + >>> my_datetime = dt.datetime(2015, 1, 1, 12, 30, 59, 0) + >>> my_datetime.isoformat(timespec='microseconds') '2015-01-01T12:30:59.000000' .. versionchanged:: 3.6 @@ -1634,8 +1665,8 @@ Instance methods: Return a string representing the date and time:: - >>> from datetime import datetime - >>> datetime(2002, 12, 4, 20, 30, 40).ctime() + >>> import datetime as dt + >>> dt.datetime(2002, 12, 4, 20, 30, 40).ctime() 'Wed Dec 4 20:30:40 2002' The output string will *not* include time zone information, regardless @@ -1665,34 +1696,34 @@ Instance methods: See also :ref:`strftime-strptime-behavior` and :meth:`datetime.isoformat`. -Examples of Usage: :class:`.datetime` +Examples of usage: :class:`!datetime` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Examples of working with :class:`.datetime` objects: .. doctest:: - >>> from datetime import datetime, date, time, timezone + >>> import datetime as dt >>> # Using datetime.combine() - >>> d = date(2005, 7, 14) - >>> t = time(12, 30) - >>> datetime.combine(d, t) + >>> d = dt.date(2005, 7, 14) + >>> t = dt.time(12, 30) + >>> dt.datetime.combine(d, t) datetime.datetime(2005, 7, 14, 12, 30) >>> # Using datetime.now() - >>> datetime.now() # doctest: +SKIP + >>> dt.datetime.now() # doctest: +SKIP datetime.datetime(2007, 12, 6, 16, 29, 43, 79043) # GMT +1 - >>> datetime.now(timezone.utc) # doctest: +SKIP + >>> dt.datetime.now(dt.timezone.utc) # doctest: +SKIP datetime.datetime(2007, 12, 6, 15, 29, 43, 79060, tzinfo=datetime.timezone.utc) >>> # Using datetime.strptime() - >>> dt = datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M") - >>> dt + >>> my_datetime = dt.datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M") + >>> my_datetime datetime.datetime(2006, 11, 21, 16, 30) >>> # Using datetime.timetuple() to get tuple of all attributes - >>> tt = dt.timetuple() + >>> tt = my_datetime.timetuple() >>> for it in tt: # doctest: +SKIP ... print(it) ... @@ -1707,7 +1738,7 @@ Examples of working with :class:`.datetime` objects: -1 # dst - method tzinfo.dst() returned None >>> # Date in ISO format - >>> ic = dt.isocalendar() + >>> ic = my_datetime.isocalendar() >>> for it in ic: # doctest: +SKIP ... print(it) ... @@ -1716,55 +1747,55 @@ Examples of working with :class:`.datetime` objects: 2 # ISO weekday >>> # Formatting a datetime - >>> dt.strftime("%A, %d. %B %Y %I:%M%p") + >>> my_datetime.strftime("%A, %d. %B %Y %I:%M%p") 'Tuesday, 21. November 2006 04:30PM' - >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(dt, "day", "month", "time") + >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(my_datetime, "day", "month", "time") 'The day is 21, the month is November, the time is 04:30PM.' The example below defines a :class:`tzinfo` subclass capturing time zone information for Kabul, Afghanistan, which used +4 UTC until 1945 and then +4:30 UTC thereafter:: - from datetime import timedelta, datetime, tzinfo, timezone + import datetime as dt - class KabulTz(tzinfo): + class KabulTz(dt.tzinfo): # Kabul used +4 until 1945, when they moved to +4:30 - UTC_MOVE_DATE = datetime(1944, 12, 31, 20, tzinfo=timezone.utc) + UTC_MOVE_DATE = dt.datetime(1944, 12, 31, 20, tzinfo=dt.timezone.utc) - def utcoffset(self, dt): - if dt.year < 1945: - return timedelta(hours=4) - elif (1945, 1, 1, 0, 0) <= dt.timetuple()[:5] < (1945, 1, 1, 0, 30): + def utcoffset(self, when): + if when.year < 1945: + return dt.timedelta(hours=4) + elif (1945, 1, 1, 0, 0) <= when.timetuple()[:5] < (1945, 1, 1, 0, 30): # An ambiguous ("imaginary") half-hour range representing # a 'fold' in time due to the shift from +4 to +4:30. - # If dt falls in the imaginary range, use fold to decide how - # to resolve. See PEP495. - return timedelta(hours=4, minutes=(30 if dt.fold else 0)) + # If when falls in the imaginary range, use fold to decide how + # to resolve. See PEP 495. + return dt.timedelta(hours=4, minutes=(30 if when.fold else 0)) else: - return timedelta(hours=4, minutes=30) + return dt.timedelta(hours=4, minutes=30) - def fromutc(self, dt): + def fromutc(self, when): # Follow same validations as in datetime.tzinfo - if not isinstance(dt, datetime): + if not isinstance(when, dt.datetime): raise TypeError("fromutc() requires a datetime argument") - if dt.tzinfo is not self: - raise ValueError("dt.tzinfo is not self") + if when.tzinfo is not self: + raise ValueError("when.tzinfo is not self") # A custom implementation is required for fromutc as # the input to this function is a datetime with utc values # but with a tzinfo set to self. # See datetime.astimezone or fromtimestamp. - if dt.replace(tzinfo=timezone.utc) >= self.UTC_MOVE_DATE: - return dt + timedelta(hours=4, minutes=30) + if when.replace(tzinfo=dt.timezone.utc) >= self.UTC_MOVE_DATE: + return when + dt.timedelta(hours=4, minutes=30) else: - return dt + timedelta(hours=4) + return when + dt.timedelta(hours=4) - def dst(self, dt): + def dst(self, when): # Kabul does not observe daylight saving time. - return timedelta(0) + return dt.timedelta(0) - def tzname(self, dt): - if dt >= self.UTC_MOVE_DATE: + def tzname(self, when): + if when >= self.UTC_MOVE_DATE: return "+04:30" return "+04" @@ -1773,17 +1804,17 @@ Usage of ``KabulTz`` from above:: >>> tz1 = KabulTz() >>> # Datetime before the change - >>> dt1 = datetime(1900, 11, 21, 16, 30, tzinfo=tz1) + >>> dt1 = dt.datetime(1900, 11, 21, 16, 30, tzinfo=tz1) >>> print(dt1.utcoffset()) 4:00:00 >>> # Datetime after the change - >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=tz1) + >>> dt2 = dt.datetime(2006, 6, 14, 13, 0, tzinfo=tz1) >>> print(dt2.utcoffset()) 4:30:00 >>> # Convert datetime to another time zone - >>> dt3 = dt2.astimezone(timezone.utc) + >>> dt3 = dt2.astimezone(dt.timezone.utc) >>> dt3 datetime.datetime(2006, 6, 14, 8, 30, tzinfo=datetime.timezone.utc) >>> dt2 @@ -1791,9 +1822,10 @@ Usage of ``KabulTz`` from above:: >>> dt2 == dt3 True + .. _datetime-time: -:class:`.time` Objects +:class:`!time` objects ---------------------- A :class:`.time` object represents a (local) time of day, independent of any particular @@ -1814,6 +1846,7 @@ day, and subject to adjustment via a :class:`tzinfo` object. If an argument outside those ranges is given, :exc:`ValueError` is raised. All default to 0 except *tzinfo*, which defaults to ``None``. + Class attributes: @@ -1872,6 +1905,7 @@ Instance attributes (read-only): .. versionadded:: 3.6 + :class:`.time` objects support equality and order comparisons, where ``a`` is considered less than ``b`` when ``a`` precedes ``b`` in time. @@ -1894,8 +1928,8 @@ In Boolean contexts, a :class:`.time` object is always considered to be true. .. versionchanged:: 3.5 Before Python 3.5, a :class:`.time` object was considered to be false if it represented midnight in UTC. This behavior was considered obscure and - error-prone and has been removed in Python 3.5. See :issue:`13936` for full - details. + error-prone and has been removed in Python 3.5. See :issue:`13936` for more + information. Other constructors: @@ -1916,22 +1950,22 @@ Other constructors: .. doctest:: - >>> from datetime import time - >>> time.fromisoformat('04:23:01') + >>> import datetime as dt + >>> dt.time.fromisoformat('04:23:01') datetime.time(4, 23, 1) - >>> time.fromisoformat('T04:23:01') + >>> dt.time.fromisoformat('T04:23:01') datetime.time(4, 23, 1) - >>> time.fromisoformat('T042301') + >>> dt.time.fromisoformat('T042301') datetime.time(4, 23, 1) - >>> time.fromisoformat('04:23:01.000384') + >>> dt.time.fromisoformat('04:23:01.000384') datetime.time(4, 23, 1, 384) - >>> time.fromisoformat('04:23:01,000384') + >>> dt.time.fromisoformat('04:23:01,000384') datetime.time(4, 23, 1, 384) - >>> time.fromisoformat('04:23:01+04:00') + >>> dt.time.fromisoformat('04:23:01+04:00') datetime.time(4, 23, 1, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) - >>> time.fromisoformat('04:23:01Z') + >>> dt.time.fromisoformat('04:23:01Z') datetime.time(4, 23, 1, tzinfo=datetime.timezone.utc) - >>> time.fromisoformat('04:23:01+00:00') + >>> dt.time.fromisoformat('04:23:01+00:00') datetime.time(4, 23, 1, tzinfo=datetime.timezone.utc) @@ -1940,6 +1974,7 @@ Other constructors: Previously, this method only supported formats that could be emitted by :meth:`time.isoformat`. + .. classmethod:: time.strptime(date_string, format) Return a :class:`.time` corresponding to *date_string*, parsed according to @@ -1964,7 +1999,7 @@ Instance methods: Return a new :class:`.time` with the same values, but with specified parameters updated. Note that ``tzinfo=None`` can be specified to create a - naive :class:`.time` from an aware :class:`.time`, without conversion of the + naive :class:`!time` from an aware :class:`!time`, without conversion of the time data. :class:`.time` objects are also supported by generic function @@ -2005,13 +2040,13 @@ Instance methods: Example:: - >>> from datetime import time - >>> time(hour=12, minute=34, second=56, microsecond=123456).isoformat(timespec='minutes') + >>> import datetime as dt + >>> dt.time(hour=12, minute=34, second=56, microsecond=123456).isoformat(timespec='minutes') '12:34' - >>> dt = time(hour=12, minute=34, second=56, microsecond=0) - >>> dt.isoformat(timespec='microseconds') + >>> my_time = dt.time(hour=12, minute=34, second=56, microsecond=0) + >>> my_time.isoformat(timespec='microseconds') '12:34:56.000000' - >>> dt.isoformat(timespec='auto') + >>> my_time.isoformat(timespec='auto') '12:34:56' .. versionchanged:: 3.6 @@ -2056,29 +2091,31 @@ Instance methods: .. versionchanged:: 3.7 The DST offset is not restricted to a whole number of minutes. + .. method:: time.tzname() If :attr:`.tzinfo` is ``None``, returns ``None``, else returns ``self.tzinfo.tzname(None)``, or raises an exception if the latter doesn't return ``None`` or a string object. -Examples of Usage: :class:`.time` + +Examples of usage: :class:`!time` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Examples of working with a :class:`.time` object:: - >>> from datetime import time, tzinfo, timedelta - >>> class TZ1(tzinfo): - ... def utcoffset(self, dt): - ... return timedelta(hours=1) - ... def dst(self, dt): - ... return timedelta(0) - ... def tzname(self,dt): + >>> import datetime as dt + >>> class TZ1(dt.tzinfo): + ... def utcoffset(self, when): + ... return dt.timedelta(hours=1) + ... def dst(self, when): + ... return dt.timedelta(0) + ... def tzname(self, when): ... return "+01:00" ... def __repr__(self): ... return f"{self.__class__.__name__}()" ... - >>> t = time(12, 10, 30, tzinfo=TZ1()) + >>> t = dt.time(12, 10, 30, tzinfo=TZ1()) >>> t datetime.time(12, 10, 30, tzinfo=TZ1()) >>> t.isoformat() @@ -2095,25 +2132,25 @@ Examples of working with a :class:`.time` object:: .. _datetime-tzinfo: -:class:`tzinfo` Objects ------------------------ +:class:`!tzinfo` objects +------------------------ .. class:: tzinfo() - This is an abstract base class, meaning that this class should not be + This is an :term:`abstract base class`, meaning that this class should not be instantiated directly. Define a subclass of :class:`tzinfo` to capture information about a particular time zone. An instance of (a concrete subclass of) :class:`tzinfo` can be passed to the constructors for :class:`.datetime` and :class:`.time` objects. The latter objects - view their attributes as being in local time, and the :class:`tzinfo` object + view their attributes as being in local time, and the :class:`!tzinfo` object supports methods revealing offset of local time from UTC, the name of the time zone, and DST offset, all relative to a date or time object passed to them. You need to derive a concrete subclass, and (at least) supply implementations of the standard :class:`tzinfo` methods needed by the :class:`.datetime` methods you use. The :mod:`!datetime` module provides - :class:`timezone`, a simple concrete subclass of :class:`tzinfo` which can + :class:`timezone`, a simple concrete subclass of :class:`!tzinfo` which can represent time zones with fixed offset from UTC such as UTC itself or North American EST and EDT. @@ -2176,31 +2213,35 @@ Examples of working with a :class:`.time` object:: ``tz.utcoffset(dt) - tz.dst(dt)`` must return the same result for every :class:`.datetime` *dt* with ``dt.tzinfo == - tz``. For sane :class:`tzinfo` subclasses, this expression yields the time + tz``. For sane :class:`!tzinfo` subclasses, this expression yields the time zone's "standard offset", which should not depend on the date or the time, but only on geographic location. The implementation of :meth:`datetime.astimezone` relies on this, but cannot detect violations; it's the programmer's - responsibility to ensure it. If a :class:`tzinfo` subclass cannot guarantee + responsibility to ensure it. If a :class:`!tzinfo` subclass cannot guarantee this, it may be able to override the default implementation of :meth:`tzinfo.fromutc` to work correctly with :meth:`~.datetime.astimezone` regardless. Most implementations of :meth:`dst` will probably look like one of these two:: - def dst(self, dt): + import datetime as dt + + def dst(self, when): # a fixed-offset class: doesn't account for DST - return timedelta(0) + return dt.timedelta(0) or:: - def dst(self, dt): + import datetime as dt + + def dst(self, when): # Code to set dston and dstoff to the time zone's DST - # transition times based on the input dt.year, and expressed + # transition times based on the input when.year, and expressed # in standard local time. - if dston <= dt.replace(tzinfo=None) < dstoff: - return timedelta(hours=1) + if dston <= when.replace(tzinfo=None) < dstoff: + return dt.timedelta(hours=1) else: - return timedelta(0) + return dt.timedelta(0) The default implementation of :meth:`dst` raises :exc:`NotImplementedError`. @@ -2217,17 +2258,17 @@ Examples of working with a :class:`.time` object:: valid replies. Return ``None`` if a string name isn't known. Note that this is a method rather than a fixed string primarily because some :class:`tzinfo` subclasses will wish to return different names depending on the specific value - of *dt* passed, especially if the :class:`tzinfo` class is accounting for + of *dt* passed, especially if the :class:`!tzinfo` class is accounting for daylight time. The default implementation of :meth:`tzname` raises :exc:`NotImplementedError`. These methods are called by a :class:`.datetime` or :class:`.time` object, in -response to their methods of the same names. A :class:`.datetime` object passes -itself as the argument, and a :class:`.time` object passes ``None`` as the +response to their methods of the same names. A :class:`!datetime` object passes +itself as the argument, and a :class:`!time` object passes ``None`` as the argument. A :class:`tzinfo` subclass's methods should therefore be prepared to -accept a *dt* argument of ``None``, or of class :class:`.datetime`. +accept a *dt* argument of ``None``, or of class :class:`!datetime`. When ``None`` is passed, it's up to the class designer to decide the best response. For example, returning ``None`` is appropriate if the class wishes to @@ -2235,10 +2276,10 @@ say that time objects don't participate in the :class:`tzinfo` protocols. It may be more useful for ``utcoffset(None)`` to return the standard UTC offset, as there is no other convention for discovering the standard offset. -When a :class:`.datetime` object is passed in response to a :class:`.datetime` +When a :class:`.datetime` object is passed in response to a :class:`!datetime` method, ``dt.tzinfo`` is the same object as *self*. :class:`tzinfo` methods can -rely on this, unless user code calls :class:`tzinfo` methods directly. The -intent is that the :class:`tzinfo` methods interpret *dt* as being in local +rely on this, unless user code calls :class:`!tzinfo` methods directly. The +intent is that the :class:`!tzinfo` methods interpret *dt* as being in local time, and not need worry about objects in other time zones. There is one more :class:`tzinfo` method that a subclass may wish to override: @@ -2266,20 +2307,22 @@ There is one more :class:`tzinfo` method that a subclass may wish to override: Skipping code for error cases, the default :meth:`fromutc` implementation acts like:: - def fromutc(self, dt): - # raise ValueError error if dt.tzinfo is not self - dtoff = dt.utcoffset() - dtdst = dt.dst() + import datetime as dt + + def fromutc(self, when): + # raise ValueError error if when.tzinfo is not self + dtoff = when.utcoffset() + dtdst = when.dst() # raise ValueError if dtoff is None or dtdst is None delta = dtoff - dtdst # this is self's standard offset if delta: - dt += delta # convert to standard local time - dtdst = dt.dst() + when += delta # convert to standard local time + dtdst = when.dst() # raise ValueError if dtdst is None if dtdst: - return dt + dtdst + return when + dtdst else: - return dt + return when In the following :download:`tzinfo_examples.py <../includes/tzinfo_examples.py>` file there are some examples of @@ -2306,9 +2349,9 @@ When DST starts (the "start" line), the local wall clock leaps from 1:59 to ``astimezone(Eastern)`` won't deliver a result with ``hour == 2`` on the day DST begins. For example, at the Spring forward transition of 2016, we get:: - >>> from datetime import datetime, timezone + >>> import datetime as dt >>> from tzinfo_examples import HOUR, Eastern - >>> u0 = datetime(2016, 3, 13, 5, tzinfo=timezone.utc) + >>> u0 = dt.datetime(2016, 3, 13, 5, tzinfo=dt.timezone.utc) >>> for i in range(4): ... u = u0 + i*HOUR ... t = u.astimezone(Eastern) @@ -2331,7 +2374,9 @@ form 5:MM and 6:MM both map to 1:MM when converted to Eastern, but earlier times have the :attr:`~.datetime.fold` attribute set to 0 and the later times have it set to 1. For example, at the Fall back transition of 2016, we get:: - >>> u0 = datetime(2016, 11, 6, 4, tzinfo=timezone.utc) + >>> import datetime as dt + >>> from tzinfo_examples import HOUR, Eastern + >>> u0 = dt.datetime(2016, 11, 6, 4, tzinfo=dt.timezone.utc) >>> for i in range(4): ... u = u0 + i*HOUR ... t = u.astimezone(Eastern) @@ -2348,7 +2393,7 @@ Note that the :class:`.datetime` instances that differ only by the value of the Applications that can't bear wall-time ambiguities should explicitly check the value of the :attr:`~.datetime.fold` attribute or avoid using hybrid :class:`tzinfo` subclasses; there are no ambiguities when using :class:`timezone`, -or any other fixed-offset :class:`tzinfo` subclass (such as a class representing +or any other fixed-offset :class:`!tzinfo` subclass (such as a class representing only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. seealso:: @@ -2371,8 +2416,8 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. _datetime-timezone: -:class:`timezone` Objects -------------------------- +:class:`!timezone` objects +-------------------------- The :class:`timezone` class is a subclass of :class:`tzinfo`, each instance of which represents a time zone defined by a fixed offset from @@ -2383,7 +2428,7 @@ locations where different offsets are used in different days of the year or where historical changes have been made to civil time. -.. class:: timezone(offset, name=None) +.. class:: timezone(offset[, name]) The *offset* argument must be specified as a :class:`timedelta` object representing the difference between the local time and UTC. It must @@ -2410,6 +2455,7 @@ where historical changes have been made to civil time. .. versionchanged:: 3.7 The UTC offset is not restricted to a whole number of minutes. + .. method:: timezone.tzname(dt) Return the fixed value specified when the :class:`timezone` instance @@ -2430,11 +2476,13 @@ where historical changes have been made to civil time. Always returns ``None``. + .. method:: timezone.fromutc(dt) Return ``dt + offset``. The *dt* argument must be an aware :class:`.datetime` instance, with ``tzinfo`` set to ``self``. + Class attributes: .. attribute:: timezone.utc @@ -2447,8 +2495,8 @@ Class attributes: .. _strftime-strptime-behavior: -:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Behavior --------------------------------------------------------------------- +:meth:`!strftime` and :meth:`!strptime` behavior +------------------------------------------------ :class:`date`, :class:`.datetime`, and :class:`.time` objects all support a ``strftime(format)`` method, to create a string representing the time under the @@ -2474,90 +2522,120 @@ versus :meth:`~.datetime.strptime`: .. _format-codes: -:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Format Codes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:meth:`!strftime` and :meth:`!strptime` format codes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ These methods accept format codes that can be used to parse and format dates:: - >>> datetime.strptime('31/01/22 23:59:59.999999', - ... '%d/%m/%y %H:%M:%S.%f') + >>> import datetime as dt + >>> dt.datetime.strptime('31/01/22 23:59:59.999999', + ... '%d/%m/%y %H:%M:%S.%f') datetime.datetime(2022, 1, 31, 23, 59, 59, 999999) >>> _.strftime('%a %d %b %Y, %I:%M%p') 'Mon 31 Jan 2022, 11:59PM' -The following is a list of all the format codes that the 1989 C standard -requires, and these work on all platforms with a standard C implementation. +The following is a list of all the format codes that the 2011 C standard +requires, and these work on all supported platforms. +-----------+--------------------------------+------------------------+-------+ -| Directive | Meaning | Example | Notes | +| Directive | Meaning | Example | Notes | +| | | | | +===========+================================+========================+=======+ -| ``%a`` | Weekday as locale's || Sun, Mon, ..., Sat | \(1) | +| ``%a`` | Weekday as locale's || Sun, Mon, ..., Sat | \(1) | | | abbreviated name. | (en_US); | | | | || So, Mo, ..., Sa | | | | | (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%A`` | Weekday as locale's full name. || Sunday, Monday, ..., | \(1) | +| ``%A`` | Weekday as locale's full name. || Sunday, Monday, ..., | \(1) | | | | Saturday (en_US); | | | | || Sonntag, Montag, ..., | | | | | Samstag (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%w`` | Weekday as a decimal number, | 0, 1, ..., 6 | | -| | where 0 is Sunday and 6 is | | | -| | Saturday. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%d`` | Day of the month as a | 01, 02, ..., 31 | \(9) | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%b`` | Month as locale's abbreviated || Jan, Feb, ..., Dec | \(1) | +| ``%b`` | Month as locale's abbreviated || Jan, Feb, ..., Dec | \(1) | | | name. | (en_US); | | | | || Jan, Feb, ..., Dez | | | | | (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%B`` | Month as locale's full name. || January, February, | \(1) | +| ``%B`` | Month as locale's full name. || January, February, | \(1) | | | | ..., December (en_US);| | | | || Januar, Februar, ..., | | | | | Dezember (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | \(9) | -| | decimal number. | | | +| ``%c`` | Locale's appropriate date and || Tue Aug 16 21:30:00 | \(1) | +| | time representation. | 1988 (en_US); | | +| | || Di 16 Aug 21:30:00 | | +| | | 1988 (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%y`` | Year without century as a | 00, 01, ..., 99 | \(9) | +| ``%C`` | The year divided by 100 and | 01, 02, ..., 99 | \(0) | +| | truncated to an integer as a | | | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%Y`` | Year with century as a decimal | 0001, 0002, ..., 2013, | \(2) | -| | number. | 2014, ..., 9998, 9999 | | +| ``%d`` | Day of the month as a | 01, 02, ..., 31 | \(9), | +| | zero-padded decimal number. | | \(10) | ++-----------+--------------------------------+------------------------+-------+ +| ``%D`` | Equivalent to ``%m/%d/%y``. | 11/28/25 | \(9) | +| | | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%e`` | The day of the month as a | ␣1, ␣2, ..., 31 | \(10) | +| | space-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%F`` | Equivalent to ``%Y-%m-%d``, | 2025-10-11, | | +| | the ISO 8601 format. | 1001-12-30 | | ++-----------+--------------------------------+------------------------+-------+ +| ``%g`` | Last 2 digits of ISO 8601 year | 00, 01, ..., 99 | \(0) | +| | representing the year that | | | +| | contains the greater part of | | | +| | the ISO week (``%V``). | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%G`` | ISO 8601 year with century | 0001, 0002, ..., 2013, | \(8) | +| | representing the year that | 2014, ..., 9998, 9999 | | +| | contains the greater part of | | | +| | the ISO week (``%V``). | | | +-----------+--------------------------------+------------------------+-------+ -| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | \(9) | +| ``%h`` | Equivalent to ``%b``. | See ``%b``. | \(0) | ++-----------+--------------------------------+------------------------+-------+ +| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | \(9) | +| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%p`` | Locale's equivalent of either || AM, PM (en_US); | \(1), | -| | AM or PM. || am, pm (de_DE) | \(3) | +| ``%j`` | Day of the year as a | 001, 002, ..., 366 | \(9) | +| | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | \(9) | +| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | \(9) | | | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4), | -| | decimal number. | | \(9) | +| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | \(9) | +| | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(5) | -| | number, zero-padded to 6 | 999999 | | -| | digits. | | | +| ``%n`` | The newline character | ``\n`` | | +| | (``'\n'``). For | | | +| | :meth:`!strptime`, zero or | | | +| | more whitespace. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%z`` | UTC offset in the form | (empty), +0000, | \(6) | -| | ``±HHMM[SS[.ffffff]]`` (empty | -0400, +1030, | | -| | string if the object is | +063415, | | -| | naive). | -030712.345216 | | +| ``%p`` | Locale's equivalent of either || AM, PM (en_US); | \(1), | +| | AM or PM. || am, pm (de_DE) | \(3) | +-----------+--------------------------------+------------------------+-------+ -| ``%Z`` | Time zone name (empty string | (empty), UTC, GMT | \(6) | -| | if the object is naive). | | | +| ``%r`` | Locale's 12-hour clock time. | 12:00:00 AM | \(1), | +| | | | \(0) | +-----------+--------------------------------+------------------------+-------+ -| ``%j`` | Day of the year as a | 001, 002, ..., 366 | \(9) | -| | zero-padded decimal number. | | | +| ``%R`` | Equivalent to ``%H:%M``. | 10:01 | | ++-----------+--------------------------------+------------------------+-------+ +| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4), | +| | decimal number. | | \(9) | ++-----------+--------------------------------+------------------------+-------+ +| ``%t`` | The tab character (``'\t'``). | ``\t`` | | +| | For :meth:`!strptime`, | | | +| | zero or more whitespace. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%T`` | ISO 8601 time format, | 10:01:59 | | +| | equivalent to ``%H:%M:%S``. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7), | +| ``%u`` | ISO 8601 weekday as a decimal | 1, 2, ..., 7 | | +| | number where 1 is Monday. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7), | | | (Sunday as the first day of | | \(9) | | | the week) as a zero-padded | | | | | decimal number. All days in a | | | @@ -2565,7 +2643,17 @@ requires, and these work on all platforms with a standard C implementation. | | Sunday are considered to be in | | | | | week 0. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7), | +| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8), | +| | number with Monday as | | \(9) | +| | the first day of the week. | | | +| | Week 01 is the week containing | | | +| | Jan 4. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%w`` | Weekday as a decimal number, | 0, 1, ..., 6 | | +| | where 0 is Sunday and 6 is | | | +| | Saturday. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7), | | | (Monday as the first day of | | \(9) | | | the week) as a zero-padded | | | | | decimal number. All days in a | | | @@ -2573,40 +2661,43 @@ requires, and these work on all platforms with a standard C implementation. | | Monday are considered to be in | | | | | week 0. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%c`` | Locale's appropriate date and || Tue Aug 16 21:30:00 | \(1) | -| | time representation. | 1988 (en_US); | | -| | || Di 16 Aug 21:30:00 | | -| | | 1988 (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%x`` | Locale's appropriate date || 08/16/88 (None); | \(1) | +| ``%x`` | Locale's appropriate date || 08/16/88 (None); | \(1) | | | representation. || 08/16/1988 (en_US); | | | | || 16.08.1988 (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%X`` | Locale's appropriate time || 21:30:00 (en_US); | \(1) | +| ``%X`` | Locale's appropriate time || 21:30:00 (en_US); | \(1) | | | representation. || 21:30:00 (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%%`` | A literal ``'%'`` character. | % | | +| ``%y`` | Year without century as a | 00, 01, ..., 99 | \(9) | +| | zero-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%Y`` | Year with century as a decimal | 0001, 0002, ..., 2013, | \(2) | +| | number. | 2014, ..., 9998, 9999 | | ++-----------+--------------------------------+------------------------+-------+ +| ``%z`` | UTC offset in the form | (empty), +0000, | \(6) | +| | ``±HHMM[SS[.ffffff]]`` (empty | -0400, +1030, | | +| | string if the object is | +063415, | | +| | naive). | -030712.345216 | | +-----------+--------------------------------+------------------------+-------+ +| ``%Z`` | Time zone name (empty string | (empty), UTC, GMT | \(6) | +| | if the object is naive). | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%%`` | A literal ``'%'`` character. | % | | ++-----------+--------------------------------+------------------------+-------+ + +The ISO 8601 year and ISO 8601 week directives are not interchangeable +with the year and week number directives above. Calling :meth:`~.datetime.strptime` with +incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. -Several additional directives not required by the C89 standard are included for -convenience. These parameters all correspond to ISO 8601 date values. +Several additional directives not required by the C11 standard are included for +convenience. +-----------+--------------------------------+------------------------+-------+ | Directive | Meaning | Example | Notes | +===========+================================+========================+=======+ -| ``%G`` | ISO 8601 year with century | 0001, 0002, ..., 2013, | \(8) | -| | representing the year that | 2014, ..., 9998, 9999 | | -| | contains the greater part of | | | -| | the ISO week (``%V``). | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%u`` | ISO 8601 weekday as a decimal | 1, 2, ..., 7 | | -| | number where 1 is Monday. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8), | -| | number with Monday as | | \(9) | -| | the first day of the week. | | | -| | Week 01 is the week containing | | | -| | Jan 4. | | | +| ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(5) | +| | number, zero-padded to 6 | 999999 | | +| | digits. | | | +-----------+--------------------------------+------------------------+-------+ | ``%:z`` | UTC offset in the form | (empty), +00:00, | \(6) | | | ``±HH:MM[:SS[.ffffff]]`` | -04:00, +10:30, | | @@ -2614,11 +2705,6 @@ convenience. These parameters all correspond to ISO 8601 date values. | | naive). | -03:07:12.345216 | | +-----------+--------------------------------+------------------------+-------+ -These may not be available on all platforms when used with the :meth:`~.datetime.strftime` -method. The ISO 8601 year and ISO 8601 week directives are not interchangeable -with the year and week number directives above. Calling :meth:`~.datetime.strptime` with -incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. - The full set of format codes supported varies across platforms, because Python calls the platform C library's :c:func:`strftime` function, and platform variations are common. To see the full set of format codes supported on your @@ -2629,18 +2715,61 @@ differences between platforms in handling of unsupported format specifiers. ``%G``, ``%u`` and ``%V`` were added. .. versionadded:: 3.12 - ``%:z`` was added. + ``%:z`` was added for :meth:`~.datetime.strftime`. + +.. versionadded:: 3.15 + ``%D``, ``%F``, ``%n``, ``%t``, and ``%:z`` were added for + :meth:`~.datetime.strptime`. -Technical Detail + +Technical detail ^^^^^^^^^^^^^^^^ Broadly speaking, ``d.strftime(fmt)`` acts like the :mod:`time` module's ``time.strftime(fmt, d.timetuple())`` although not all objects support a :meth:`~date.timetuple` method. -For the :meth:`.datetime.strptime` class method, the default value is -``1900-01-01T00:00:00.000``: any components not specified in the format string -will be pulled from the default value. [#]_ +For the :meth:`.datetime.strptime` and :meth:`.date.strptime` class methods, +the default value is ``1900-01-01T00:00:00.000``: any components not specified +in the format string will be pulled from the default value. + +.. note:: + Format strings without separators can be ambiguous for parsing. For + example, with ``%Y%m%d``, the string ``2026111`` may be parsed either as + ``2026-11-01`` or as ``2026-01-11``. + Use separators to ensure the input is parsed as intended. + +.. note:: + When used to parse partial dates lacking a year, :meth:`.datetime.strptime` + and :meth:`.date.strptime` will raise when encountering February 29 because + the default year of 1900 is *not* a leap year. Always add a default leap + year to partial date strings before parsing. + +.. testsetup:: + + # doctest seems to turn the warning into an error which makes it + # show up and require matching and prevents the actual interesting + # exception from being raised. + # Manually apply the catch_warnings context manager + import warnings + catch_warnings = warnings.catch_warnings() + catch_warnings.__enter__() + warnings.simplefilter("ignore") + +.. testcleanup:: + + catch_warnings.__exit__() + +.. doctest:: + + >>> import datetime as dt + >>> value = "2/29" + >>> dt.datetime.strptime(value, "%m/%d") + Traceback (most recent call last): + ... + ValueError: day 29 must be in range 1..28 for month 2 in year 1900 + >>> dt.datetime.strptime(f"1904 {value}", "%Y %m/%d") + datetime.datetime(1904, 2, 29, 0, 0) Using ``datetime.strptime(date_string, format)`` is equivalent to:: @@ -2666,6 +2795,9 @@ an empty string instead. Notes: +(0) + This format code is currently unsupported by :meth:`~.datetime.strptime`. + (1) Because the format depends on the current locale, care should be taken when making assumptions about the output value. Field orderings will vary (for @@ -2724,12 +2856,18 @@ Notes: When the ``%z`` directive is provided to the :meth:`~.datetime.strptime` method, the UTC offsets can have a colon as a separator between hours, minutes and seconds. - For example, ``'+01:00:00'`` will be parsed as an offset of one hour. - In addition, providing ``'Z'`` is identical to ``'+00:00'``. + For example, both ``'+010000'`` and ``'+01:00:00'`` will be parsed as an offset + of one hour. In addition, providing ``'Z'`` is identical to ``'+00:00'``. ``%:z`` - Behaves exactly as ``%z``, but has a colon separator added between - hours, minutes and seconds. + When used with :meth:`~.datetime.strftime`, behaves exactly as ``%z``, + except that a colon separator is added between hours, minutes and seconds. + + When used with :meth:`~.datetime.strptime`, the UTC offset is *required* + to have a colon as a separator between hours, minutes and seconds. + For example, ``'+01:00:00'`` (but *not* ``'+010000'``) will be parsed as + an offset of one hour. In addition, providing ``'Z'`` is identical to + ``'+00:00'``. ``%Z`` In :meth:`~.datetime.strftime`, ``%Z`` is replaced by an empty string if @@ -2771,23 +2909,24 @@ Notes: include a year in the format. If the value you need to parse lacks a year, append an explicit dummy leap year. Otherwise your code will raise an exception when it encounters leap day because the default year used by the - parser is not a leap year. Users run into this bug every four years... + parser (1900) is not a leap year. Users run into that bug every leap year. .. doctest:: >>> month_day = "02/29" - >>> datetime.strptime(f"{month_day};1984", "%m/%d;%Y") # No leap year bug. + >>> dt.datetime.strptime(f"{month_day};1984", "%m/%d;%Y") # No leap year bug. datetime.datetime(1984, 2, 29, 0, 0) - .. deprecated-removed:: 3.13 3.15 + .. versionchanged:: 3.15 + Using ``%d`` without a year now raises :exc:`ValueError`. + + .. deprecated-removed:: 3.15 3.17 :meth:`~.datetime.strptime` calls using a format string containing - a day of month without a year now emit a - :exc:`DeprecationWarning`. In 3.15 or later we may change this into - an error or change the default year to a leap year. See :gh:`70647`. + ``%e`` without a year now emit a :exc:`DeprecationWarning`. .. rubric:: Footnotes -.. [#] If, that is, we ignore the effects of Relativity +.. [#] If, that is, we ignore the effects of relativity. .. [#] This matches the definition of the "proleptic Gregorian" calendar in Dershowitz and Reingold's book *Calendrical Calculations*, @@ -2798,5 +2937,3 @@ Notes: .. [#] See R. H. van Gent's `guide to the mathematics of the ISO 8601 calendar `_ for a good explanation. - -.. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since 1900 is not a leap year. diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 39e287b15214e4a..646981e8692cc53 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -8,7 +8,7 @@ -------------- -:mod:`dbm` is a generic interface to variants of the DBM database: +:mod:`!dbm` is a generic interface to variants of the DBM database: * :mod:`dbm.sqlite3` * :mod:`dbm.gnu` @@ -90,10 +90,13 @@ the Oracle Berkeley DB. .. versionchanged:: 3.11 *file* accepts a :term:`path-like object`. -The object returned by :func:`~dbm.open` supports the same basic functionality as a -:class:`dict`; keys and their corresponding values can be stored, retrieved, and -deleted, and the :keyword:`in` operator and the :meth:`!keys` method are -available, as well as :meth:`!get` and :meth:`!setdefault` methods. +The object returned by :func:`~dbm.open` supports the basic +functionality of mutable :term:`mappings `; +keys and their corresponding values can be stored, retrieved, and +deleted, and iteration, the :keyword:`in` operator and methods :meth:`!keys`, +:meth:`!get`, :meth:`!setdefault` and :meth:`!clear` are available. +The :meth:`!keys` method returns a list instead of a view object. +The :meth:`!setdefault` method requires two arguments. Key and values are always stored as :class:`bytes`. This means that when strings are used they are implicitly converted to the default encoding before @@ -104,7 +107,7 @@ will automatically close them when done. .. versionchanged:: 3.2 :meth:`!get` and :meth:`!setdefault` methods are now available for all - :mod:`dbm` backends. + :mod:`!dbm` backends. .. versionchanged:: 3.4 Added native support for the context management protocol to the objects @@ -114,6 +117,10 @@ will automatically close them when done. Deleting a key from a read-only database raises a database module specific exception instead of :exc:`KeyError`. +.. versionchanged:: 3.13 + :meth:`!clear` methods are now available for all :mod:`!dbm` backends. + + The following example records some hostnames and a corresponding title, and then prints out the contents of the database:: @@ -150,11 +157,10 @@ then prints out the contents of the database:: The individual submodules are described in the following sections. -:mod:`dbm.sqlite3` --- SQLite backend for dbm ---------------------------------------------- +:mod:`!dbm.sqlite3` --- SQLite backend for dbm +---------------------------------------------- .. module:: dbm.sqlite3 - :platform: All :synopsis: SQLite backend for dbm .. versionadded:: 3.13 @@ -164,8 +170,8 @@ The individual submodules are described in the following sections. -------------- This module uses the standard library :mod:`sqlite3` module to provide an -SQLite backend for the :mod:`dbm` module. -The files created by :mod:`dbm.sqlite3` can thus be opened by :mod:`sqlite3`, +SQLite backend for the :mod:`!dbm` module. +The files created by :mod:`!dbm.sqlite3` can thus be opened by :mod:`sqlite3`, or any other SQLite browser, including the SQLite CLI. .. include:: ../includes/wasm-notavail.rst @@ -173,9 +179,6 @@ or any other SQLite browser, including the SQLite CLI. .. function:: open(filename, /, flag="r", mode=0o666) Open an SQLite database. - The returned object behaves like a :term:`mapping`, - implements a :meth:`!close` method, - and supports a "closing" context manager via the :keyword:`with` keyword. :param filename: The path to the database to be opened. @@ -192,6 +195,17 @@ or any other SQLite browser, including the SQLite CLI. The Unix file access mode of the file (default: octal ``0o666``), used only when the database has to be created. + The returned database object behaves similar to a mutable :term:`mapping`, + but the :meth:`!keys` method returns a list, and + the :meth:`!setdefault` method requires two arguments. + It also supports a "closing" context manager via the :keyword:`with` keyword. + + The following methods are also provided: + + .. method:: sqlite3.close() + + Close the SQLite database. + .. method:: sqlite3.reorganize() If you have carried out a lot of deletions and would like to shrink the space @@ -200,38 +214,45 @@ or any other SQLite browser, including the SQLite CLI. .. note:: While reorganizing, as much as two times the size of the original database is required - in free disk space. However, be aware that this factor changes for each :mod:`dbm` submodule. + in free disk space. However, be aware that this factor changes for each :mod:`!dbm` submodule. + + .. versionadded:: 3.15 - .. versionadded:: next -:mod:`dbm.gnu` --- GNU database manager ---------------------------------------- +:mod:`!dbm.gnu` --- GNU database manager +---------------------------------------- .. module:: dbm.gnu - :platform: Unix :synopsis: GNU database manager **Source code:** :source:`Lib/dbm/gnu.py` -------------- -The :mod:`dbm.gnu` module provides an interface to the :abbr:`GDBM (GNU dbm)` +The :mod:`!dbm.gnu` module provides an interface to the :abbr:`GDBM (GNU dbm)` library, similar to the :mod:`dbm.ndbm` module, but with additional functionality like crash tolerance. .. note:: - The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible + The file formats created by :mod:`!dbm.gnu` and :mod:`dbm.ndbm` are incompatible and can not be used interchangeably. .. include:: ../includes/wasm-mobile-notavail.rst +.. availability:: Unix. + .. exception:: error - Raised on :mod:`dbm.gnu`-specific errors, such as I/O errors. :exc:`KeyError` is + Raised on :mod:`!dbm.gnu`-specific errors, such as I/O errors. :exc:`KeyError` is raised for general mapping errors like specifying an incorrect key. +.. data:: open_flags + + A string of characters the *flag* parameter of :meth:`~dbm.gnu.open` supports. + + .. function:: open(filename, flag="r", mode=0o666, /) Open a GDBM database and return a :class:`!gdbm` object. @@ -254,9 +275,6 @@ functionality like crash tolerance. * ``'s'``: Synchronized mode. Changes to the database will be written immediately to the file. * ``'u'``: Do not lock database. - * ``'m'``: Do not use :manpage:`mmap(2)`. - This may harm performance, but improve crash tolerance. - .. versionadded:: next Not all flags are valid for all versions of GDBM. See the :data:`open_flags` member for a list of supported flag characters. @@ -270,14 +288,25 @@ functionality like crash tolerance. .. versionchanged:: 3.11 *filename* accepts a :term:`path-like object`. - .. data:: open_flags + :class:`!gdbm` objects behave similar to mutable :term:`mappings `, + but methods :meth:`!items`, :meth:`!values`, :meth:`!pop`, :meth:`!popitem`, + and :meth:`!update` are not supported, + the :meth:`!keys` method returns a list, and + the :meth:`!setdefault` method requires two arguments. + It also supports a "closing" context manager via the :keyword:`with` keyword. + + .. versionchanged:: 3.2 + Added the :meth:`!get` and :meth:`!setdefault` methods. - A string of characters the *flag* parameter of :meth:`~dbm.gnu.open` supports. + .. versionchanged:: 3.13 + Added the :meth:`!clear` method. - :class:`!gdbm` objects behave similar to :term:`mappings `, - but :meth:`!items` and :meth:`!values` methods are not supported. The following methods are also provided: + .. method:: gdbm.close() + + Close the GDBM database. + .. method:: gdbm.firstkey() It's possible to loop over every key in the database using this method and the @@ -306,43 +335,32 @@ functionality like crash tolerance. .. note:: While reorganizing, as much as one time the size of the original database is required - in free disk space. However, be aware that this factor changes for each :mod:`dbm` submodule. + in free disk space. However, be aware that this factor changes for each :mod:`!dbm` submodule. .. method:: gdbm.sync() When the database has been opened in fast mode, this method forces any unwritten data to be written to the disk. - .. method:: gdbm.close() - - Close the GDBM database. - - .. method:: gdbm.clear() - Remove all items from the GDBM database. - - .. versionadded:: 3.13 - - -:mod:`dbm.ndbm` --- New Database Manager ----------------------------------------- +:mod:`!dbm.ndbm` --- New Database Manager +----------------------------------------- .. module:: dbm.ndbm - :platform: Unix :synopsis: The New Database Manager **Source code:** :source:`Lib/dbm/ndbm.py` -------------- -The :mod:`dbm.ndbm` module provides an interface to the +The :mod:`!dbm.ndbm` module provides an interface to the :abbr:`NDBM (New Database Manager)` library. This module can be used with the "classic" NDBM interface or the :abbr:`GDBM (GNU dbm)` compatibility interface. .. note:: - The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible + The file formats created by :mod:`dbm.gnu` and :mod:`!dbm.ndbm` are incompatible and can not be used interchangeably. .. warning:: @@ -354,9 +372,11 @@ This module can be used with the "classic" NDBM interface or the .. include:: ../includes/wasm-mobile-notavail.rst +.. availability:: Unix. + .. exception:: error - Raised on :mod:`dbm.ndbm`-specific errors, such as I/O errors. :exc:`KeyError` is raised + Raised on :mod:`!dbm.ndbm`-specific errors, such as I/O errors. :exc:`KeyError` is raised for general mapping errors like specifying an incorrect key. @@ -383,26 +403,31 @@ This module can be used with the "classic" NDBM interface or the :param int mode: |mode_param_doc| - :class:`!ndbm` objects behave similar to :term:`mappings `, - but :meth:`!items` and :meth:`!values` methods are not supported. - The following methods are also provided: - .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. - .. method:: ndbm.close() + :class:`!ndbm` objects behave similar to mutable :term:`mappings `, + but methods :meth:`!items`, :meth:`!values`, :meth:`!pop`, :meth:`!popitem`, + and :meth:`!update` are not supported, + the :meth:`!keys` method returns a list, and + the :meth:`!setdefault` method requires two arguments. + It also supports a "closing" context manager via the :keyword:`with` keyword. - Close the NDBM database. + .. versionchanged:: 3.2 + Added the :meth:`!get` and :meth:`!setdefault` methods. - .. method:: ndbm.clear() + .. versionchanged:: 3.13 + Added the :meth:`!clear` method. - Remove all items from the NDBM database. + The following method is also provided: + + .. method:: ndbm.close() - .. versionadded:: 3.13 + Close the NDBM database. -:mod:`dbm.dumb` --- Portable DBM implementation ------------------------------------------------ +:mod:`!dbm.dumb` --- Portable DBM implementation +------------------------------------------------ .. module:: dbm.dumb :synopsis: Portable implementation of the simple DBM interface. @@ -413,32 +438,29 @@ This module can be used with the "classic" NDBM interface or the .. note:: - The :mod:`dbm.dumb` module is intended as a last resort fallback for the - :mod:`dbm` module when a more robust module is not available. The :mod:`dbm.dumb` + The :mod:`!dbm.dumb` module is intended as a last resort fallback for the + :mod:`!dbm` module when a more robust module is not available. The :mod:`!dbm.dumb` module is not written for speed and is not nearly as heavily used as the other database modules. -------------- -The :mod:`dbm.dumb` module provides a persistent :class:`dict`-like +The :mod:`!dbm.dumb` module provides a persistent :class:`dict`-like interface which is written entirely in Python. -Unlike other :mod:`dbm` backends, such as :mod:`dbm.gnu`, no +Unlike other :mod:`!dbm` backends, such as :mod:`dbm.gnu`, no external library is required. The :mod:`!dbm.dumb` module defines the following: .. exception:: error - Raised on :mod:`dbm.dumb`-specific errors, such as I/O errors. :exc:`KeyError` is + Raised on :mod:`!dbm.dumb`-specific errors, such as I/O errors. :exc:`KeyError` is raised for general mapping errors like specifying an incorrect key. .. function:: open(filename, flag="c", mode=0o666) Open a :mod:`!dbm.dumb` database. - The returned database object behaves similar to a :term:`mapping`, - in addition to providing :meth:`~dumbdbm.sync` and :meth:`~dumbdbm.close` - methods. :param filename: The basename of the database file (without extensions). @@ -463,7 +485,7 @@ The :mod:`!dbm.dumb` module defines the following: Python's AST compiler. .. warning:: - :mod:`dbm.dumb` does not support concurrent read/write access. (Multiple + :mod:`!dbm.dumb` does not support concurrent read/write access. (Multiple simultaneous read accesses are safe.) When a program has the database open for writing, no other program should have it open for reading or writing. @@ -477,14 +499,12 @@ The :mod:`!dbm.dumb` module defines the following: .. versionchanged:: 3.11 *filename* accepts a :term:`path-like object`. - In addition to the methods provided by the - :class:`collections.abc.MutableMapping` class, - the following methods are provided: - - .. method:: dumbdbm.sync() + The returned database object behaves similar to a mutable :term:`mapping`, + but the :meth:`!keys` and :meth:`!items` methods return lists, and + the :meth:`!setdefault` method requires two arguments. + It also supports a "closing" context manager via the :keyword:`with` keyword. - Synchronize the on-disk directory and data files. This method is called - by the :meth:`shelve.Shelf.sync` method. + The following methods are also provided: .. method:: dumbdbm.close() @@ -498,6 +518,11 @@ The :mod:`!dbm.dumb` module defines the following: .. note:: While reorganizing, no additional free disk space is required. However, be aware - that this factor changes for each :mod:`dbm` submodule. + that this factor changes for each :mod:`!dbm` submodule. - .. versionadded:: next + .. versionadded:: 3.15 + + .. method:: dumbdbm.sync() + + Synchronize the on-disk directory and data files. This method is called + by the :meth:`shelve.Shelf.sync` method. diff --git a/Doc/library/debug.rst b/Doc/library/debug.rst index 60223657a440434..f87c2481fb89cdb 100644 --- a/Doc/library/debug.rst +++ b/Doc/library/debug.rst @@ -1,5 +1,5 @@ *********************** -Debugging and Profiling +Debugging and profiling *********************** These libraries help you with Python development: the debugger enables you to @@ -15,7 +15,8 @@ intrusive debugging or patching. bdb.rst faulthandler.rst pdb.rst - profile.rst + profiling.rst + pstats.rst timeit.rst trace.rst tracemalloc.rst diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 10ddfa02b431563..2af5dfce9612b37 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -4,14 +4,6 @@ .. module:: decimal :synopsis: Implementation of the General Decimal Arithmetic Specification. -.. moduleauthor:: Eric Price -.. moduleauthor:: Facundo Batista -.. moduleauthor:: Raymond Hettinger -.. moduleauthor:: Aahz -.. moduleauthor:: Tim Peters -.. moduleauthor:: Stefan Krah -.. sectionauthor:: Raymond D. Hettinger - **Source code:** :source:`Lib/decimal.py` .. import modules for testing inline doctests with the Sphinx doctest builder @@ -30,14 +22,16 @@ -------------- -The :mod:`decimal` module provides support for fast correctly rounded +The :mod:`!decimal` module provides support for fast correctly rounded decimal floating-point arithmetic. It offers several advantages over the :class:`float` datatype: -* Decimal "is based on a floating-point model which was designed with people - in mind, and necessarily has a paramount guiding principle -- computers must - provide an arithmetic that works in the same way as the arithmetic that - people learn at school." -- excerpt from the decimal arithmetic specification. +* Decimal "is based on a `floating-point model + `__ which was designed + with people in mind, and necessarily has a paramount guiding principle -- + computers must provide an arithmetic that works in the same way as the + arithmetic that people learn at school." -- excerpt from the decimal + arithmetic specification. * Decimal numbers can be represented exactly. In contrast, numbers like ``1.1`` and ``2.2`` do not have exact representations in binary @@ -238,6 +232,26 @@ floating-point flying circus: >>> c % a Decimal('0.77') +Decimals can be formatted (with :func:`format` built-in or :ref:`f-strings`) in +fixed-point or scientific notation, using the same formatting syntax (see +:ref:`formatspec`) as builtin :class:`float` type: + +.. doctest:: + + >>> format(Decimal('2.675'), "f") + '2.675' + >>> format(Decimal('2.675'), ".2f") + '2.68' + >>> f"{Decimal('2.675'):.2f}" + '2.68' + >>> format(Decimal('2.675'), ".2e") + '2.68e+0' + >>> with localcontext() as ctx: + ... ctx.rounding = ROUND_DOWN + ... print(format(Decimal('2.675'), ".2f")) + ... + 2.67 + And some mathematical functions are also available to Decimal: >>> getcontext().prec = 28 @@ -264,10 +278,10 @@ allows the settings to be changed. This approach meets the needs of most applications. For more advanced work, it may be useful to create alternate contexts using the -Context() constructor. To make an alternate active, use the :func:`setcontext` +:meth:`Context` constructor. To make an alternate active, use the :func:`setcontext` function. -In accordance with the standard, the :mod:`decimal` module provides two ready to +In accordance with the standard, the :mod:`!decimal` module provides two ready to use standard contexts, :const:`BasicContext` and :const:`ExtendedContext`. The former is especially useful for debugging because many of the traps are enabled: @@ -573,7 +587,7 @@ Decimal objects >>> Decimal(321).exp() Decimal('2.561702493119680037517373933E+139') - .. classmethod:: from_float(f) + .. classmethod:: from_float(f, /) Alternative constructor that only accepts instances of :class:`float` or :class:`int`. @@ -600,7 +614,7 @@ Decimal objects .. versionadded:: 3.1 - .. classmethod:: from_number(number) + .. classmethod:: from_number(number, /) Alternative constructor that only accepts instances of :class:`float`, :class:`int` or :class:`Decimal`, but not strings @@ -991,7 +1005,7 @@ Each thread has its own current context which is accessed or changed using the Return the current context for the active thread. -.. function:: setcontext(c) +.. function:: setcontext(c, /) Set the current context for the active thread to *c*. @@ -1168,11 +1182,11 @@ In addition to the three supplied contexts, new contexts can be created with the Return a duplicate of the context. - .. method:: copy_decimal(num) + .. method:: copy_decimal(num, /) Return a copy of the Decimal instance num. - .. method:: create_decimal(num) + .. method:: create_decimal(num='0', /) Creates a new Decimal instance from *num* but using *self* as context. Unlike the :class:`Decimal` constructor, the context precision, @@ -1196,7 +1210,7 @@ In addition to the three supplied contexts, new contexts can be created with the If the argument is a string, no leading or trailing whitespace or underscores are permitted. - .. method:: create_decimal_from_float(f) + .. method:: create_decimal_from_float(f, /) Creates a new Decimal instance from a float *f* but rounding using *self* as the context. Unlike the :meth:`Decimal.from_float` class method, @@ -1234,222 +1248,222 @@ In addition to the three supplied contexts, new contexts can be created with the recounted here. - .. method:: abs(x) + .. method:: abs(x, /) Returns the absolute value of *x*. - .. method:: add(x, y) + .. method:: add(x, y, /) Return the sum of *x* and *y*. - .. method:: canonical(x) + .. method:: canonical(x, /) Returns the same Decimal object *x*. - .. method:: compare(x, y) + .. method:: compare(x, y, /) Compares *x* and *y* numerically. - .. method:: compare_signal(x, y) + .. method:: compare_signal(x, y, /) Compares the values of the two operands numerically. - .. method:: compare_total(x, y) + .. method:: compare_total(x, y, /) Compares two operands using their abstract representation. - .. method:: compare_total_mag(x, y) + .. method:: compare_total_mag(x, y, /) Compares two operands using their abstract representation, ignoring sign. - .. method:: copy_abs(x) + .. method:: copy_abs(x, /) Returns a copy of *x* with the sign set to 0. - .. method:: copy_negate(x) + .. method:: copy_negate(x, /) Returns a copy of *x* with the sign inverted. - .. method:: copy_sign(x, y) + .. method:: copy_sign(x, y, /) Copies the sign from *y* to *x*. - .. method:: divide(x, y) + .. method:: divide(x, y, /) Return *x* divided by *y*. - .. method:: divide_int(x, y) + .. method:: divide_int(x, y, /) Return *x* divided by *y*, truncated to an integer. - .. method:: divmod(x, y) + .. method:: divmod(x, y, /) Divides two numbers and returns the integer part of the result. - .. method:: exp(x) + .. method:: exp(x, /) Returns ``e ** x``. - .. method:: fma(x, y, z) + .. method:: fma(x, y, z, /) Returns *x* multiplied by *y*, plus *z*. - .. method:: is_canonical(x) + .. method:: is_canonical(x, /) Returns ``True`` if *x* is canonical; otherwise returns ``False``. - .. method:: is_finite(x) + .. method:: is_finite(x, /) Returns ``True`` if *x* is finite; otherwise returns ``False``. - .. method:: is_infinite(x) + .. method:: is_infinite(x, /) Returns ``True`` if *x* is infinite; otherwise returns ``False``. - .. method:: is_nan(x) + .. method:: is_nan(x, /) Returns ``True`` if *x* is a qNaN or sNaN; otherwise returns ``False``. - .. method:: is_normal(x) + .. method:: is_normal(x, /) Returns ``True`` if *x* is a normal number; otherwise returns ``False``. - .. method:: is_qnan(x) + .. method:: is_qnan(x, /) Returns ``True`` if *x* is a quiet NaN; otherwise returns ``False``. - .. method:: is_signed(x) + .. method:: is_signed(x, /) Returns ``True`` if *x* is negative; otherwise returns ``False``. - .. method:: is_snan(x) + .. method:: is_snan(x, /) Returns ``True`` if *x* is a signaling NaN; otherwise returns ``False``. - .. method:: is_subnormal(x) + .. method:: is_subnormal(x, /) Returns ``True`` if *x* is subnormal; otherwise returns ``False``. - .. method:: is_zero(x) + .. method:: is_zero(x, /) Returns ``True`` if *x* is a zero; otherwise returns ``False``. - .. method:: ln(x) + .. method:: ln(x, /) Returns the natural (base e) logarithm of *x*. - .. method:: log10(x) + .. method:: log10(x, /) Returns the base 10 logarithm of *x*. - .. method:: logb(x) + .. method:: logb(x, /) Returns the exponent of the magnitude of the operand's MSD. - .. method:: logical_and(x, y) + .. method:: logical_and(x, y, /) Applies the logical operation *and* between each operand's digits. - .. method:: logical_invert(x) + .. method:: logical_invert(x, /) Invert all the digits in *x*. - .. method:: logical_or(x, y) + .. method:: logical_or(x, y, /) Applies the logical operation *or* between each operand's digits. - .. method:: logical_xor(x, y) + .. method:: logical_xor(x, y, /) Applies the logical operation *xor* between each operand's digits. - .. method:: max(x, y) + .. method:: max(x, y, /) Compares two values numerically and returns the maximum. - .. method:: max_mag(x, y) + .. method:: max_mag(x, y, /) Compares the values numerically with their sign ignored. - .. method:: min(x, y) + .. method:: min(x, y, /) Compares two values numerically and returns the minimum. - .. method:: min_mag(x, y) + .. method:: min_mag(x, y, /) Compares the values numerically with their sign ignored. - .. method:: minus(x) + .. method:: minus(x, /) Minus corresponds to the unary prefix minus operator in Python. - .. method:: multiply(x, y) + .. method:: multiply(x, y, /) Return the product of *x* and *y*. - .. method:: next_minus(x) + .. method:: next_minus(x, /) Returns the largest representable number smaller than *x*. - .. method:: next_plus(x) + .. method:: next_plus(x, /) Returns the smallest representable number larger than *x*. - .. method:: next_toward(x, y) + .. method:: next_toward(x, y, /) Returns the number closest to *x*, in direction towards *y*. - .. method:: normalize(x) + .. method:: normalize(x, /) Reduces *x* to its simplest form. - .. method:: number_class(x) + .. method:: number_class(x, /) Returns an indication of the class of *x*. - .. method:: plus(x) + .. method:: plus(x, /) Plus corresponds to the unary prefix plus operator in Python. This operation applies the context precision and rounding, so it is *not* an @@ -1490,7 +1504,7 @@ In addition to the three supplied contexts, new contexts can be created with the always exact. - .. method:: quantize(x, y) + .. method:: quantize(x, y, /) Returns a value equal to *x* (rounded), having the exponent of *y*. @@ -1500,7 +1514,7 @@ In addition to the three supplied contexts, new contexts can be created with the Just returns 10, as this is Decimal, :) - .. method:: remainder(x, y) + .. method:: remainder(x, y, /) Returns the remainder from integer division. @@ -1508,43 +1522,43 @@ In addition to the three supplied contexts, new contexts can be created with the dividend. - .. method:: remainder_near(x, y) + .. method:: remainder_near(x, y, /) Returns ``x - y * n``, where *n* is the integer nearest the exact value of ``x / y`` (if the result is 0 then its sign will be the sign of *x*). - .. method:: rotate(x, y) + .. method:: rotate(x, y, /) Returns a rotated copy of *x*, *y* times. - .. method:: same_quantum(x, y) + .. method:: same_quantum(x, y, /) Returns ``True`` if the two operands have the same exponent. - .. method:: scaleb (x, y) + .. method:: scaleb (x, y, /) Returns the first operand after adding the second value its exp. - .. method:: shift(x, y) + .. method:: shift(x, y, /) Returns a shifted copy of *x*, *y* times. - .. method:: sqrt(x) + .. method:: sqrt(x, /) Square root of a non-negative number to context precision. - .. method:: subtract(x, y) + .. method:: subtract(x, y, /) Return the difference between *x* and *y*. - .. method:: to_eng_string(x) + .. method:: to_eng_string(x, /) Convert to a string, using engineering notation if an exponent is needed. @@ -1553,12 +1567,12 @@ In addition to the three supplied contexts, new contexts can be created with the require the addition of either one or two trailing zeros. - .. method:: to_integral_exact(x) + .. method:: to_integral_exact(x, /) Rounds to an integer. - .. method:: to_sci_string(x) + .. method:: to_sci_string(x, /) Converts a number to a string using scientific notation. @@ -1569,7 +1583,16 @@ In addition to the three supplied contexts, new contexts can be created with the Constants --------- -The constants in this section are only relevant for the C module. They +.. data:: SPEC_VERSION + + The highest version of the General Decimal Arithmetic + Specification that this implementation complies with. + See https://speleotrove.com/decimal/decarith.html for the specification. + + .. versionadded:: 3.15 + + +The following constants are only relevant for the C module. They are also included in the pure Python version for compatibility. +---------------------------------+---------------------+-------------------------------+ @@ -1816,7 +1839,7 @@ properties of addition: >>> u * (v+w) Decimal('0.0060000') -The :mod:`decimal` module makes it possible to restore the identities by +The :mod:`!decimal` module makes it possible to restore the identities by expanding the precision sufficiently to avoid loss of significance: .. doctest:: newcontext @@ -1838,7 +1861,7 @@ expanding the precision sufficiently to avoid loss of significance: Special values ^^^^^^^^^^^^^^ -The number system for the :mod:`decimal` module provides special values +The number system for the :mod:`!decimal` module provides special values including ``NaN``, ``sNaN``, ``-Infinity``, ``Infinity``, and two zeros, ``+0`` and ``-0``. @@ -2100,20 +2123,20 @@ to work with the :class:`Decimal` class:: Decimal FAQ ----------- -Q. It is cumbersome to type ``decimal.Decimal('1234.5')``. Is there a way to +Q: It is cumbersome to type ``decimal.Decimal('1234.5')``. Is there a way to minimize typing when using the interactive interpreter? -A. Some users abbreviate the constructor to just a single letter: +A: Some users abbreviate the constructor to just a single letter: >>> D = decimal.Decimal >>> D('1.23') + D('3.45') Decimal('4.68') -Q. In a fixed-point application with two decimal places, some inputs have many +Q: In a fixed-point application with two decimal places, some inputs have many places and need to be rounded. Others are not supposed to have excess digits and need to be validated. What methods should be used? -A. The :meth:`~Decimal.quantize` method rounds to a fixed number of decimal places. If +A: The :meth:`~Decimal.quantize` method rounds to a fixed number of decimal places. If the :const:`Inexact` trap is set, it is also useful for validation: >>> TWOPLACES = Decimal(10) ** -2 # same as Decimal('0.01') @@ -2131,10 +2154,10 @@ the :const:`Inexact` trap is set, it is also useful for validation: ... Inexact: None -Q. Once I have valid two place inputs, how do I maintain that invariant +Q: Once I have valid two place inputs, how do I maintain that invariant throughout an application? -A. Some operations like addition, subtraction, and multiplication by an integer +A: Some operations like addition, subtraction, and multiplication by an integer will automatically preserve fixed point. Others operations, like division and non-integer multiplication, will change the number of decimal places and need to be followed-up with a :meth:`~Decimal.quantize` step: @@ -2166,21 +2189,21 @@ to handle the :meth:`~Decimal.quantize` step: >>> div(b, a) Decimal('0.03') -Q. There are many ways to express the same value. The numbers ``200``, +Q: There are many ways to express the same value. The numbers ``200``, ``200.000``, ``2E2``, and ``.02E+4`` all have the same value at various precisions. Is there a way to transform them to a single recognizable canonical value? -A. The :meth:`~Decimal.normalize` method maps all equivalent values to a single +A: The :meth:`~Decimal.normalize` method maps all equivalent values to a single representative: >>> values = map(Decimal, '200 200.000 2E2 .02E+4'.split()) >>> [v.normalize() for v in values] [Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2')] -Q. When does rounding occur in a computation? +Q: When does rounding occur in a computation? -A. It occurs *after* the computation. The philosophy of the decimal +A: It occurs *after* the computation. The philosophy of the decimal specification is that numbers are considered exact and are created independent of the current context. They can even have greater precision than current context. Computations process with those @@ -2198,10 +2221,10 @@ applied to the *result* of the computation:: >>> pi + 0 - Decimal('0.00005'). # Intermediate values are rounded Decimal('3.1416') -Q. Some decimal values always print with exponential notation. Is there a way +Q: Some decimal values always print with exponential notation. Is there a way to get a non-exponential representation? -A. For some values, exponential notation is the only way to express the number +A: For some values, exponential notation is the only way to express the number of significant places in the coefficient. For example, expressing ``5.0E+3`` as ``5000`` keeps the value constant but cannot show the original's two-place significance. @@ -2216,9 +2239,9 @@ value unchanged: >>> remove_exponent(Decimal('5E+3')) Decimal('5000') -Q. Is there a way to convert a regular float to a :class:`Decimal`? +Q: Is there a way to convert a regular float to a :class:`Decimal`? -A. Yes, any binary floating-point number can be exactly expressed as a +A: Yes, any binary floating-point number can be exactly expressed as a Decimal though an exact conversion may take more precision than intuition would suggest: @@ -2227,19 +2250,19 @@ suggest: >>> Decimal(math.pi) Decimal('3.141592653589793115997963468544185161590576171875') -Q. Within a complex calculation, how can I make sure that I haven't gotten a +Q: Within a complex calculation, how can I make sure that I haven't gotten a spurious result because of insufficient precision or rounding anomalies. -A. The decimal module makes it easy to test results. A best practice is to +A: The decimal module makes it easy to test results. A best practice is to re-run calculations using greater precision and with various rounding modes. Widely differing results indicate insufficient precision, rounding mode issues, ill-conditioned inputs, or a numerically unstable algorithm. -Q. I noticed that context precision is applied to the results of operations but +Q: I noticed that context precision is applied to the results of operations but not to the inputs. Is there anything to watch out for when mixing values of different precisions? -A. Yes. The principle is that all values are considered to be exact and so is +A: Yes. The principle is that all values are considered to be exact and so is the arithmetic on those values. Only the results are rounded. The advantage for inputs is that "what you type is what you get". A disadvantage is that the results can look odd if you forget that the inputs haven't been rounded: @@ -2267,9 +2290,9 @@ Alternatively, inputs can be rounded upon creation using the >>> Context(prec=5, rounding=ROUND_DOWN).create_decimal('1.2345678') Decimal('1.2345') -Q. Is the CPython implementation fast for large numbers? +Q: Is the CPython implementation fast for large numbers? -A. Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of +A: Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of the decimal module integrate the high speed `libmpdec `_ library for arbitrary precision correctly rounded decimal floating-point arithmetic [#]_. diff --git a/Doc/library/dialog.rst b/Doc/library/dialog.rst index e0693e8eb6ed226..402cfcfe75369f8 100644 --- a/Doc/library/dialog.rst +++ b/Doc/library/dialog.rst @@ -1,31 +1,45 @@ -Tkinter Dialogs +Tkinter dialogs =============== -:mod:`tkinter.simpledialog` --- Standard Tkinter input dialogs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:mod:`!tkinter.simpledialog` --- Standard Tkinter input dialogs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: tkinter.simpledialog - :platform: Tk :synopsis: Simple dialog windows **Source code:** :source:`Lib/tkinter/simpledialog.py` -------------- -The :mod:`tkinter.simpledialog` module contains convenience classes and +The :mod:`!tkinter.simpledialog` module contains convenience classes and functions for creating simple modal dialogs to get a value from the user. -.. function:: askfloat(title, prompt, **kw) - askinteger(title, prompt, **kw) - askstring(title, prompt, **kw) +.. function:: askfloat(title, prompt, *, initialvalue=None, minvalue=None, maxvalue=None, parent=None) + askinteger(title, prompt, *, initialvalue=None, minvalue=None, maxvalue=None, parent=None) + askstring(title, prompt, *, initialvalue=None, show=None, parent=None) - The above three functions provide dialogs that prompt the user to enter a value - of the desired type. + Prompt the user to enter a value of the desired type and return it, or + ``None`` if the dialog is cancelled. + + *title* is the dialog title and *prompt* the message shown above the entry. + *initialvalue* is the value initially placed in the entry. + *parent* is the window over which the dialog is shown. + :func:`askinteger` and :func:`askfloat` also accept *minvalue* and + *maxvalue*, which bound the accepted value. + :func:`askstring` also accepts *show*, a character used to mask the entered + text, for example ``'*'`` to hide a password. .. class:: Dialog(parent, title=None) The base class for custom dialogs. + Instantiating it shows the dialog modally and returns once the user closes + it; the entered value is then available in the :attr:`!result` attribute. + + .. attribute:: result + + The value produced by :meth:`apply`, or ``None`` if the dialog was + cancelled. .. method:: body(master) @@ -37,23 +51,59 @@ functions for creating simple modal dialogs to get a value from the user. Default behaviour adds OK and Cancel buttons. Override for custom button layouts. + .. method:: validate() + + Validate the data entered by the user. + Return true if it is valid, in which case the dialog proceeds to + :meth:`apply`; return false to keep the dialog open. + The default implementation always returns true; override it to check the + input. + + .. method:: apply() + + Process the data entered by the user, for example by storing it in the + :attr:`!result` attribute. + Called after :meth:`validate` succeeds and just before the dialog is + destroyed. + The default implementation does nothing; override it to act on or store + the result. + + .. method:: destroy() + + Destroy the dialog window, clearing the reference to the widget that had + the initial focus. -:mod:`tkinter.filedialog` --- File selection dialogs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. class:: SimpleDialog(master, text='', buttons=[], default=None, cancel=None, title=None, class_=None) + + A simple modal dialog that displays the message *text* above a row of push + buttons whose labels are given by *buttons*, and returns the index of the + button the user presses. + *default* is the index of the button activated by the Return key, *cancel* + the index returned when the window is closed through the window manager, + *title* the window title, and *class_* the Tk class name of the window. + + .. method:: go() + + Display the dialog, wait until the user presses a button or closes the + window, and return the index of the chosen button. + + + +:mod:`!tkinter.filedialog` --- File selection dialogs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: tkinter.filedialog - :platform: Tk :synopsis: Dialog classes for file selection **Source code:** :source:`Lib/tkinter/filedialog.py` -------------- -The :mod:`tkinter.filedialog` module provides classes and factory functions for +The :mod:`!tkinter.filedialog` module provides classes and factory functions for creating file/directory selection windows. -Native Load/Save Dialogs +Native load/save dialogs ------------------------ The following classes and functions provide file dialog windows that combine a @@ -79,34 +129,46 @@ listed below: **Static factory functions** The below functions when called create a modal, native look-and-feel dialog, -wait for the user's selection, then return the selected value(s) or ``None`` to the -caller. +wait for the user's selection, and return it. +The exact return value depends on the function (see below); when the dialog is +cancelled it is an empty string, an empty tuple, an empty list or ``None``. .. function:: askopenfile(mode="r", **options) askopenfiles(mode="r", **options) - The above two functions create an :class:`Open` dialog and return the opened - file object(s) in read-only mode. + Create an :class:`Open` dialog. + :func:`askopenfile` returns the opened file object, or ``None`` if the + dialog is cancelled. + :func:`askopenfiles` returns a list of the opened file objects, or an empty + list if cancelled. + The files are opened in mode *mode* (read-only ``'r'`` by default). .. function:: asksaveasfile(mode="w", **options) - Create a :class:`SaveAs` dialog and return a file object opened in write-only mode. + Create a :class:`SaveAs` dialog and return the opened file object, or + ``None`` if the dialog is cancelled. + The file is opened in mode *mode* (``'w'`` by default). .. function:: askopenfilename(**options) askopenfilenames(**options) - The above two functions create an :class:`Open` dialog and return the - selected filename(s) that correspond to existing file(s). + Create an :class:`Open` dialog. + :func:`askopenfilename` returns the selected filename as a string, or an + empty string if the dialog is cancelled. + :func:`askopenfilenames` returns a tuple of the selected filenames, or an + empty tuple if cancelled. .. function:: asksaveasfilename(**options) - Create a :class:`SaveAs` dialog and return the selected filename. + Create a :class:`SaveAs` dialog and return the selected filename as a + string, or an empty string if the dialog is cancelled. .. function:: askdirectory(**options) - | Prompt user to select a directory. - | Additional keyword option: - | *mustexist* - determines if selection must be an existing directory. + Prompt the user to select a directory, and return its path as a string, or + an empty string if the dialog is cancelled. + Additional keyword option: *mustexist* - if true, the user may only select + an existing directory (false by default). .. class:: Open(master=None, **options) SaveAs(master=None, **options) @@ -170,6 +232,13 @@ These do not emulate the native look-and-feel of the platform. Exit dialog returning current selection. + .. method:: ok_command() + + Called when the user confirms the current selection. + The base implementation accepts the selection and closes the dialog; + :class:`LoadFileDialog` and :class:`SaveFileDialog` override it to check + the selection first. + .. method:: quit(how=None) Exit dialog returning filename, if any. @@ -204,18 +273,17 @@ These do not emulate the native look-and-feel of the platform. directory. Confirmation is required if an already existing file is selected. -:mod:`tkinter.commondialog` --- Dialog window templates -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:mod:`!tkinter.commondialog` --- Dialog window templates +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: tkinter.commondialog - :platform: Tk :synopsis: Tkinter base class for dialogs **Source code:** :source:`Lib/tkinter/commondialog.py` -------------- -The :mod:`tkinter.commondialog` module provides the :class:`Dialog` class that +The :mod:`!tkinter.commondialog` module provides the :class:`Dialog` class that is the base class for dialogs defined in other supporting modules. .. class:: Dialog(master=None, **options) @@ -225,6 +293,40 @@ is the base class for dialogs defined in other supporting modules. Render the Dialog window. +:mod:`!tkinter.dialog` --- Classic Tk dialog boxes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. module:: tkinter.dialog + :synopsis: A simple dialog box built on the classic Tk widgets. + +**Source code:** :source:`Lib/tkinter/dialog.py` + +-------------- + +The :mod:`!tkinter.dialog` module provides a simple modal dialog box built on +the classic (non-themed) Tk widgets. + +.. data:: DIALOG_ICON + + The name of the default bitmap (``'questhead'``) displayed by a + :class:`Dialog`. + +.. class:: Dialog(master=None, cnf={}, **kw) + + Display a modal dialog box built from the classic (non-themed) Tk widgets + and wait for the user to press one of its buttons. + The options, given through *cnf* or as keyword arguments, include *title* + (the window title), *text* (the message), *bitmap* (an icon, + :data:`DIALOG_ICON` by default), *default* (the index of the default button) + and *strings* (the sequence of button labels). + After construction, the :attr:`!num` attribute holds the index of the button + the user pressed. + + .. method:: destroy() + + Destroy the dialog window. + + .. seealso:: Modules :mod:`tkinter.messagebox`, :ref:`tut-files` diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index ce948a6860f02c1..25edb40e35a630a 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -4,10 +4,6 @@ .. module:: difflib :synopsis: Helpers for computing differences between objects. -.. moduleauthor:: Tim Peters -.. sectionauthor:: Tim Peters -.. Markup by Fred L. Drake, Jr. - **Source code:** :source:`Lib/difflib.py` .. testsetup:: @@ -231,7 +227,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. *linejunk*: A function that accepts a single string argument, and returns true if the string is junk, or false if not. The default is ``None``. There is also a module-level function :func:`IS_LINE_JUNK`, which filters out lines - without visible characters, except for at most one pound character (``'#'``) + without visible characters, except for at most one hash character (``'#'``) -- however the underlying :class:`SequenceMatcher` class does a dynamic analysis of which lines are so frequent as to constitute noise, and this usually works better than using this function. @@ -278,7 +274,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. emu -.. function:: unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n') +.. function:: unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n', *, color=False) Compare *a* and *b* (lists of strings); return a delta (a :term:`generator` generating the delta lines) in unified diff format. @@ -297,6 +293,10 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. For inputs that do not have trailing newlines, set the *lineterm* argument to ``""`` so that the output will be uniformly newline free. + Set *color* to ``True`` to enable output in color, similar to + :program:`git diff --color`. Even if enabled, it can be + :ref:`controlled using environment variables `. + The unified diff format normally has a header for filenames and modification times. Any or all of these may be specified using strings for *fromfile*, *tofile*, *fromfiledate*, and *tofiledate*. The modification times are normally @@ -319,6 +319,10 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. See :ref:`difflib-interface` for a more detailed example. + .. versionchanged:: 3.15 + Added the *color* parameter. + + .. function:: diff_bytes(dfunc, a, b, fromfile=b'', tofile=b'', fromfiledate=b'', tofiledate=b'', n=3, lineterm=b'\n') Compare *a* and *b* (lists of bytes objects) using *dfunc*; yield a @@ -351,14 +355,14 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. .. seealso:: - `Pattern Matching: The Gestalt Approach `_ + `Pattern Matching: The Gestalt Approach `_ Discussion of a similar algorithm by John W. Ratcliff and D. E. Metzener. This - was published in `Dr. Dobb's Journal `_ in July, 1988. + was published in Dr. Dobb's Journal in July, 1988. .. _sequence-matcher: -SequenceMatcher Objects +SequenceMatcher objects ----------------------- The :class:`SequenceMatcher` class has this constructor: @@ -586,7 +590,7 @@ are always at least as large as :meth:`~SequenceMatcher.ratio`: .. _sequencematcher-examples: -SequenceMatcher Examples +SequenceMatcher examples ------------------------ This example compares two strings, considering blanks to be "junk": @@ -637,7 +641,7 @@ If you want to know how to change the first sequence into the second, use .. _differ-objects: -Differ Objects +Differ objects -------------- Note that :class:`Differ`\ -generated deltas make no claim to be **minimal** @@ -686,7 +690,7 @@ The :class:`Differ` class has this constructor: .. _differ-examples: -Differ Example +Differ example -------------- This example compares two texts. First we set up the texts, sequences of @@ -720,7 +724,7 @@ Finally, we compare the two: >>> result = list(d.compare(text1, text2)) -``result`` is a list of strings, so let's pretty-print it: +``result`` is a list of strings, so let's pretty-print it:: >>> from pprint import pprint >>> pprint(result) @@ -735,7 +739,7 @@ Finally, we compare the two: '? ++++ ^ ^\n', '+ 5. Flat is better than nested.\n'] -As a single multi-line string it looks like this: +As a single multi-line string it looks like this:: >>> import sys >>> sys.stdout.writelines(result) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index ac8a911c40a8606..3e7ae509fedcea6 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -14,7 +14,7 @@ -------------- -The :mod:`dis` module supports the analysis of CPython :term:`bytecode` by +The :mod:`!dis` module supports the analysis of CPython :term:`bytecode` by disassembling it. The CPython bytecode which this module takes as an input is defined in the file :file:`Include/opcode.h` and used by the compiler and the interpreter. @@ -38,7 +38,7 @@ interpreter. Some instructions are accompanied by one or more inline cache entries, which take the form of :opcode:`CACHE` instructions. These instructions are hidden by default, but can be shown by passing ``show_caches=True`` to - any :mod:`dis` utility. Furthermore, the interpreter now adapts the + any :mod:`!dis` utility. Furthermore, the interpreter now adapts the bytecode to specialize it for different runtime conditions. The adaptive bytecode can be shown by passing ``adaptive=True``. @@ -87,7 +87,7 @@ the following command can be used to display the disassembly of Command-line interface ---------------------- -The :mod:`dis` module can be invoked as a script from the command line: +The :mod:`!dis` module can be invoked as a script from the command line: .. code-block:: sh @@ -223,7 +223,7 @@ Example: Analysis functions ------------------ -The :mod:`dis` module also defines the following analysis functions that convert +The :mod:`!dis` module also defines the following analysis functions that convert the input directly to the desired output. They can be useful if only a single operation is being performed, so the intermediate analysis object isn't useful: @@ -400,7 +400,7 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.10 The :pep:`626` :meth:`~codeobject.co_lines` method is used instead of the - :attr:`~codeobject.co_firstlineno` and :attr:`~codeobject.co_lnotab` + :attr:`~codeobject.co_firstlineno` and :attr:`!codeobject.co_lnotab` attributes of the :ref:`code object `. .. versionchanged:: 3.13 @@ -585,6 +585,22 @@ operations on it as if it was a Python list. The top of the stack corresponds to generate line tracing events. +.. opcode:: NOT_TAKEN + + Do nothing code. + Used by the interpreter to record :monitoring-event:`BRANCH_LEFT` + and :monitoring-event:`BRANCH_RIGHT` events for :mod:`sys.monitoring`. + + .. versionadded:: 3.14 + + +.. opcode:: POP_ITER + + Removes the iterator from the top of the stack. + + .. versionadded:: 3.14 + + .. opcode:: POP_TOP Removes the top-of-stack item:: @@ -752,7 +768,7 @@ not have to be) the original ``STACK[-2]``. end = STACK.pop() start = STACK.pop() container = STACK.pop() - values = STACK.pop() + value = STACK.pop() container[start:end] = value .. versionadded:: 3.12 @@ -1122,8 +1138,8 @@ iterations of the loop. .. opcode:: BUILD_TEMPLATE - Constructs a new :class:`~string.templatelib.Template` from a tuple - of strings and a tuple of interpolations and pushes the resulting instance + Constructs a new :class:`~string.templatelib.Template` instance from a tuple + of strings and a tuple of interpolations and pushes the resulting object onto the stack:: interpolations = STACK.pop() @@ -1135,8 +1151,8 @@ iterations of the loop. .. opcode:: BUILD_INTERPOLATION (format) - Constructs a new :class:`~string.templatelib.Interpolation` from a - value and its source expression and pushes the resulting instance onto the + Constructs a new :class:`~string.templatelib.Interpolation` instance from a + value and its source expression and pushes the resulting object onto the stack. If no conversion or format specification is present, ``format`` is set to @@ -1626,7 +1642,7 @@ iterations of the loop. Pushes a ``NULL`` to the stack. Used in the call sequence to match the ``NULL`` pushed by - :opcode:`LOAD_METHOD` for non-method calls. + :opcode:`LOAD_ATTR` for non-method calls. .. versionadded:: 3.11 @@ -1657,9 +1673,13 @@ iterations of the loop. * ``0x02`` a dictionary of keyword-only parameters' default values * ``0x04`` a tuple of strings containing parameters' annotations * ``0x08`` a tuple containing cells for free variables, making a closure + * ``0x10`` the :term:`annotate function` for the function object .. versionadded:: 3.13 + .. versionchanged:: 3.14 + Added ``0x10`` to indicate the annotate function for the function object. + .. opcode:: BUILD_SLICE (argc) @@ -1807,7 +1827,7 @@ iterations of the loop. ignore it. Before, only opcodes ``>= HAVE_ARGUMENT`` had an argument. .. versionchanged:: 3.12 - Pseudo instructions were added to the :mod:`dis` module, and for them + Pseudo instructions were added to the :mod:`!dis` module, and for them it is not true that comparison with ``HAVE_ARGUMENT`` indicates whether they use their arg. @@ -1947,14 +1967,15 @@ but are replaced by real opcodes or removed before bytecode is generated. Marks the end of the code block associated with the last ``SETUP_FINALLY``, ``SETUP_CLEANUP`` or ``SETUP_WITH``. + .. opcode:: JUMP -.. opcode:: JUMP_NO_INTERRUPT + JUMP_NO_INTERRUPT Undirected relative jump instructions which are replaced by their directed (forward/backward) counterparts by the assembler. .. opcode:: JUMP_IF_TRUE -.. opcode:: JUMP_IF_FALSE + JUMP_IF_FALSE Conditional jumps which do not impact the stack. Replaced by the sequence ``COPY 1``, ``TO_BOOL``, ``POP_JUMP_IF_TRUE/FALSE``. @@ -1970,12 +1991,6 @@ but are replaced by real opcodes or removed before bytecode is generated. This opcode is now a pseudo-instruction. -.. opcode:: LOAD_METHOD - - Optimized unbound method lookup. Emitted as a ``LOAD_ATTR`` opcode - with a flag set in the arg. - - .. _opcode_collections: Opcode collections diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 5a2c6bdd27c386a..3298697af8511b6 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -4,16 +4,11 @@ .. module:: doctest :synopsis: Test pieces of code within docstrings. -.. moduleauthor:: Tim Peters -.. sectionauthor:: Tim Peters -.. sectionauthor:: Moshe Zadka -.. sectionauthor:: Edward Loper - **Source code:** :source:`Lib/doctest.py` -------------- -The :mod:`doctest` module searches for pieces of text that look like interactive +The :mod:`!doctest` module searches for pieces of text that look like interactive Python sessions, and then executes those sessions to verify that they work exactly as shown. There are several common ways to use doctest: @@ -85,7 +80,7 @@ Here's a complete but small example module:: import doctest doctest.testmod() -If you run :file:`example.py` directly from the command line, :mod:`doctest` +If you run :file:`example.py` directly from the command line, :mod:`!doctest` works its magic: .. code-block:: shell-session @@ -94,7 +89,7 @@ works its magic: $ There's no output! That's normal, and it means all the examples worked. Pass -``-v`` to the script, and :mod:`doctest` prints a detailed log of what +``-v`` to the script, and :mod:`!doctest` prints a detailed log of what it's trying, and prints a summary at the end: .. code-block:: shell-session @@ -130,7 +125,7 @@ And so on, eventually ending with: Test passed. $ -That's all you need to know to start making productive use of :mod:`doctest`! +That's all you need to know to start making productive use of :mod:`!doctest`! Jump in. The following sections provide full details. Note that there are many examples of doctests in the standard Python test suite and libraries. Especially useful examples can be found in the standard test file @@ -252,7 +247,7 @@ For more information on :func:`testfile`, see section :ref:`doctest-basic-api`. Command-line Usage ------------------ -The :mod:`doctest` module can be invoked as a script from the command line: +The :mod:`!doctest` module can be invoked as a script from the command line: .. code-block:: bash @@ -350,6 +345,13 @@ searches them recursively for docstrings, which are then scanned for tests. Any classes found are recursively searched similarly, to test docstrings in their contained methods and nested classes. +.. note:: + + ``doctest`` can only automatically discover classes and functions that are + defined at the module level or inside other classes. + + Since nested classes and functions only exist when an outer function + is called, they cannot be discovered. Define them outside to make them visible. .. _doctest-finding-examples: @@ -443,7 +445,7 @@ The fine print: What's the Execution Context? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -By default, each time :mod:`doctest` finds a docstring to test, it uses a +By default, each time :mod:`!doctest` finds a docstring to test, it uses a *shallow copy* of :mod:`!M`'s globals, so that running tests doesn't change the module's real globals, and so that one test in :mod:`!M` can't leave behind crumbs that accidentally allow another test to work. This means examples can @@ -723,7 +725,7 @@ The second group of options controls how test failures are reported: There is also a way to register new option flag names, though this isn't -useful unless you intend to extend :mod:`doctest` internals via subclassing: +useful unless you intend to extend :mod:`!doctest` internals via subclassing: .. function:: register_optionflag(name) @@ -826,7 +828,7 @@ disabling an option via ``-`` in a directive can be useful. Warnings ^^^^^^^^ -:mod:`doctest` is serious about requiring exact matches in expected output. If +:mod:`!doctest` is serious about requiring exact matches in expected output. If even a single character doesn't match, the test fails. This will probably surprise you a few times, as you learn exactly what Python does and doesn't guarantee about output. For example, when printing a set, Python doesn't @@ -1028,7 +1030,7 @@ Unittest API ------------ As your collection of doctest'ed modules grows, you'll want a way to run all -their doctests systematically. :mod:`doctest` provides two functions that can +their doctests systematically. :mod:`!doctest` provides two functions that can be used to create :mod:`unittest` test suites from modules and text files containing doctests. To integrate with :mod:`unittest` test discovery, include a :ref:`load_tests ` function in your test module:: @@ -1116,7 +1118,7 @@ from text files and modules with doctests: The global ``__file__`` is added to the globals provided to doctests loaded from a text file using :func:`DocFileSuite`. - .. versionchanged:: next + .. versionchanged:: 3.15 Run each example as a :ref:`subtest `. @@ -1157,7 +1159,7 @@ from text files and modules with doctests: :func:`DocTestSuite` returns an empty :class:`unittest.TestSuite` if *module* contains no docstrings instead of raising :exc:`ValueError`. - .. versionchanged:: next + .. versionchanged:: 3.15 Run each example as a :ref:`subtest `. Under the covers, :func:`DocTestSuite` creates a :class:`unittest.TestSuite` out @@ -1172,7 +1174,7 @@ of :class:`!DocTestCase`. So both ways of creating a :class:`unittest.TestSuite` run instances of :class:`!DocTestCase`. This is important for a subtle reason: when you run -:mod:`doctest` functions yourself, you can control the :mod:`!doctest` options in +:mod:`!doctest` functions yourself, you can control the :mod:`!doctest` options in use directly, by passing option flags to :mod:`!doctest` functions. However, if you're writing a :mod:`unittest` framework, :mod:`!unittest` ultimately controls when and how tests get run. The framework author typically wants to control @@ -1180,13 +1182,13 @@ when and how tests get run. The framework author typically wants to control options), but there's no way to pass options through :mod:`!unittest` to :mod:`!doctest` test runners. -For this reason, :mod:`doctest` also supports a notion of :mod:`!doctest` +For this reason, :mod:`!doctest` also supports a notion of :mod:`!doctest` reporting flags specific to :mod:`unittest` support, via this function: .. function:: set_unittest_reportflags(flags) - Set the :mod:`doctest` reporting flags to use. + Set the :mod:`!doctest` reporting flags to use. Argument *flags* takes the :ref:`bitwise OR ` of option flags. See section :ref:`doctest-options`. Only "reporting flags" can be used. @@ -1557,7 +1559,7 @@ DocTestRunner objects containing *example*. *out* is the output function that was passed to :meth:`DocTestRunner.run`. - .. versionadded:: next + .. versionadded:: 3.15 .. method:: report_start(out, test, example) @@ -1916,7 +1918,7 @@ There are two exceptions that may be raised by :class:`DebugRunner` instances: Soapbox ------- -As mentioned in the introduction, :mod:`doctest` has grown to have three primary +As mentioned in the introduction, :mod:`!doctest` has grown to have three primary uses: #. Checking examples in docstrings. @@ -1934,7 +1936,7 @@ this that needs to be learned---it may not be natural at first. Examples should add genuine value to the documentation. A good example can often be worth many words. If done with care, the examples will be invaluable for your users, and will pay back the time it takes to collect them many times over as the years go -by and things change. I'm still amazed at how often one of my :mod:`doctest` +by and things change. I'm still amazed at how often one of my :mod:`!doctest` examples stops working after a "harmless" change. Doctest also makes an excellent tool for regression testing, especially if you diff --git a/Doc/library/email.charset.rst b/Doc/library/email.charset.rst index 6875af2be49d7a9..76a57031862c85b 100644 --- a/Doc/library/email.charset.rst +++ b/Doc/library/email.charset.rst @@ -19,7 +19,7 @@ registry and several convenience methods for manipulating this registry. Instances of :class:`Charset` are used in several other modules within the :mod:`email` package. -Import this class from the :mod:`email.charset` module. +Import this class from the :mod:`!email.charset` module. .. class:: Charset(input_charset=DEFAULT_CHARSET) @@ -164,7 +164,7 @@ Import this class from the :mod:`email.charset` module. This method allows you to compare two :class:`Charset` instances for inequality. -The :mod:`email.charset` module also provides the following functions for adding +The :mod:`!email.charset` module also provides the following functions for adding new entries to the global character set, alias, and codec registries: diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst index b33fe82a6e4c9fc..04a41667f7dc2cd 100644 --- a/Doc/library/email.contentmanager.rst +++ b/Doc/library/email.contentmanager.rst @@ -4,9 +4,6 @@ .. module:: email.contentmanager :synopsis: Storing and Retrieving Content from MIME Parts -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray - **Source code:** :source:`Lib/email/contentmanager.py` ------------ diff --git a/Doc/library/email.encoders.rst b/Doc/library/email.encoders.rst index 9c8c8c9234ed7a7..1a9a1cad3a619e8 100644 --- a/Doc/library/email.encoders.rst +++ b/Doc/library/email.encoders.rst @@ -25,7 +25,7 @@ is especially true for :mimetype:`image/\*` and :mimetype:`text/\*` type message containing binary data. The :mod:`email` package provides some convenient encoders in its -:mod:`~email.encoders` module. These encoders are actually used by the +:mod:`!encoders` module. These encoders are actually used by the :class:`~email.mime.audio.MIMEAudio` and :class:`~email.mime.image.MIMEImage` class constructors to provide default encodings. All encoder functions take exactly one argument, the message object to encode. They usually extract the diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index 689e7397cbcf1f2..2f7c9140cfcbe55 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -8,7 +8,7 @@ -------------- -The following exception classes are defined in the :mod:`email.errors` module: +The following exception classes are defined in the :mod:`!email.errors` module: .. exception:: MessageError() diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index a3132d02687bc9e..6f4f813a0f84d8c 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -232,7 +232,7 @@ a formatted string representation of a message object. For more detail, see :mod:`email.message`. -The :mod:`email.generator` module also provides a derived class, +The :mod:`!email.generator` module also provides a derived class, :class:`DecodedGenerator`, which is like the :class:`Generator` base class, except that non-\ :mimetype:`text` parts are not serialized, but are instead represented in the output stream by a string derived from a template filled diff --git a/Doc/library/email.header.rst b/Doc/library/email.header.rst index f49885b87852357..e7e21d036e07deb 100644 --- a/Doc/library/email.header.rst +++ b/Doc/library/email.header.rst @@ -28,13 +28,13 @@ transferred using only 7-bit ASCII characters, so a slew of RFCs have been written describing how to encode email containing non-ASCII characters into :rfc:`2822`\ -compliant format. These RFCs include :rfc:`2045`, :rfc:`2046`, :rfc:`2047`, and :rfc:`2231`. The :mod:`email` package supports these standards -in its :mod:`email.header` and :mod:`email.charset` modules. +in its :mod:`!email.header` and :mod:`email.charset` modules. If you want to include non-ASCII characters in your email headers, say in the :mailheader:`Subject` or :mailheader:`To` fields, you should use the :class:`Header` class and assign the field in the :class:`~email.message.Message` object to an instance of :class:`Header` instead of using a string for the header -value. Import the :class:`Header` class from the :mod:`email.header` module. +value. Import the :class:`Header` class from the :mod:`!email.header` module. For example:: >>> from email.message import Message @@ -170,7 +170,7 @@ Here is the :class:`Header` class description: This method allows you to compare two :class:`Header` instances for inequality. -The :mod:`email.header` module also provides the following convenient functions. +The :mod:`!email.header` module also provides the following convenient functions. .. function:: decode_header(header) diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index 7f8044932fae993..619c17c98e8d89c 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -4,9 +4,6 @@ .. module:: email.headerregistry :synopsis: Automatic Parsing of headers based on the field name -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray - **Source code:** :source:`Lib/email/headerregistry.py` -------------- @@ -96,9 +93,10 @@ headers. ``kwds`` is a dictionary containing one pre-initialized key, ``defects``. ``defects`` is an empty list. The parse method should append any detected defects to this list. On return, the ``kwds`` dictionary *must* contain - values for at least the keys ``decoded`` and ``defects``. ``decoded`` - should be the string value for the header (that is, the header value fully - decoded to unicode). The parse method should assume that *string* may + values for at least the keys ``decoded``, ``defects`` and ``parse_tree``. + ``decoded`` should be the string value for the header (that is, the header + value fully decoded to unicode). ``parse_tree`` is set to the parse tree obtained + from parsing the header. The parse method should assume that *string* may contain content-transfer-encoded parts, but should correctly handle all valid unicode characters as well so that it can parse un-encoded header values. @@ -269,6 +267,10 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1. A dictionary mapping parameter names to parameter values. + .. versionchanged:: 3.15 + It is now a :class:`frozendict` instead of a + :class:`types.MappingProxyType`. + .. class:: ContentTypeHeader @@ -294,7 +296,7 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1. ``inline`` and ``attachment`` are the only valid values in common use. -.. class:: ContentTransferEncoding +.. class:: ContentTransferEncodingHeader Handles the :mailheader:`Content-Transfer-Encoding` header. diff --git a/Doc/library/email.iterators.rst b/Doc/library/email.iterators.rst index 090981d84b4de3d..ed300cdb30fdd69 100644 --- a/Doc/library/email.iterators.rst +++ b/Doc/library/email.iterators.rst @@ -10,7 +10,7 @@ Iterating over a message object tree is fairly easy with the :meth:`Message.walk ` method. The -:mod:`email.iterators` module provides some useful higher level iterations over +:mod:`!email.iterators` module provides some useful higher level iterations over message object trees. diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index 71d6e321f387bcb..b70df130e06dfa5 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -3,9 +3,6 @@ .. module:: email.message :synopsis: The base class representing email messages. -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray , - Barry A. Warsaw **Source code:** :source:`Lib/email/message.py` @@ -14,7 +11,7 @@ .. versionadded:: 3.6 [1]_ The central class in the :mod:`email` package is the :class:`EmailMessage` -class, imported from the :mod:`email.message` module. It is the base class for +class, imported from the :mod:`!email.message` module. It is the base class for the :mod:`email` object model. :class:`EmailMessage` provides the core functionality for setting and querying header fields, for accessing message bodies, and for creating or modifying structured messages. @@ -57,7 +54,7 @@ message objects. :class:`~email.policy.default` policy, which follows the rules of the email RFCs except for line endings (instead of the RFC mandated ``\r\n``, it uses the Python standard ``\n`` line endings). For more information see the - :mod:`~email.policy` documentation. + :mod:`~email.policy` documentation. [2]_ .. method:: as_string(unixfrom=False, maxheaderlen=None, policy=None) @@ -749,3 +746,9 @@ message objects. .. [1] Originally added in 3.4 as a :term:`provisional module `. Docs for legacy message class moved to :ref:`compat32_message`. + +.. [2] The :class:`EmailMessage` class requires a policy that provides a + ``content_manager`` attribute for content management methods like + ``set_content()`` and ``get_content()`` to work. The legacy + :const:`~email.policy.compat32` policy does not support these methods + and should not be used with :class:`EmailMessage`. diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index 6a70714dc3ee42b..6a67bf7c8e555dd 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -125,10 +125,10 @@ Here is the API for the :class:`BytesFeedParser`: Parser API ^^^^^^^^^^ -The :class:`BytesParser` class, imported from the :mod:`email.parser` module, +The :class:`BytesParser` class, imported from the :mod:`!email.parser` module, provides an API that can be used to parse a message when the complete contents of the message are available in a :term:`bytes-like object` or file. The -:mod:`email.parser` module also provides :class:`Parser` for parsing strings, +:mod:`!email.parser` module also provides :class:`Parser` for parsing strings, and header-only parsers, :class:`BytesHeaderParser` and :class:`HeaderParser`, which can be used if you're only interested in the headers of the message. :class:`BytesHeaderParser` and :class:`HeaderParser` @@ -155,7 +155,7 @@ message body, instead setting the payload to the raw body. Read all the data from the binary file-like object *fp*, parse the resulting bytes, and return the message object. *fp* must support - both the :meth:`~io.IOBase.readline` and the :meth:`~io.IOBase.read` + both the :meth:`~io.IOBase.readline` and the :meth:`~io.BufferedIOBase.read` methods. The bytes contained in *fp* must be formatted as a block of :rfc:`5322` diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index 6b997ee784f6e44..816d02d86f4fc4b 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -4,9 +4,6 @@ .. module:: email.policy :synopsis: Controlling the parsing and generating of messages -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray - .. versionadded:: 3.3 **Source code:** :source:`Lib/email/policy.py` @@ -406,11 +403,26 @@ added matters. To illustrate:: .. attribute:: utf8 If ``False``, follow :rfc:`5322`, supporting non-ASCII characters in - headers by encoding them as "encoded words". If ``True``, follow - :rfc:`6532` and use ``utf-8`` encoding for headers. Messages + headers by encoding them as :rfc:`2047` "encoded words". If ``True``, + follow :rfc:`6532` and use ``utf-8`` encoding for headers. Messages formatted in this way may be passed to SMTP servers that support the ``SMTPUTF8`` extension (:rfc:`6531`). + When ``False``, the generator will raise + :exc:`~email.errors.HeaderWriteError` if any header includes non-ASCII + characters in a context where :rfc:`2047` does not permit encoded words. + This particularly applies to mailboxes ("addr-spec") with non-ASCII + characters, which can be created via + :class:`~email.headerregistry.Address`. To use a mailbox with a non-ASCII + domain name with ``utf8=False``, first encode the domain using the + third-party :pypi:`idna` or :pypi:`uts46` module or with + :mod:`encodings.idna`. It is not possible to use a non-ASCII username + ("local-part") in a mailbox when ``utf8=False``. + + .. versionchanged:: 3.15 + Can trigger the raising of :exc:`~email.errors.HeaderWriteError`. + (Earlier versions incorrectly applied :rfc:`2047` in certain contexts, + mostly notably in addr-specs.) .. attribute:: refold_source @@ -602,7 +614,7 @@ The header objects and their attributes are described in This concrete :class:`Policy` is the backward compatibility policy. It replicates the behavior of the email package in Python 3.2. The - :mod:`~email.policy` module also defines an instance of this class, + :mod:`!policy` module also defines an instance of this class, :const:`compat32`, that is used as the default policy. Thus the default behavior of the email package is to maintain compatibility with Python 3.2. @@ -662,6 +674,13 @@ The header objects and their attributes are described in An instance of :class:`Compat32`, providing backward compatibility with the behavior of the email package in Python 3.2. + .. note:: + + The :const:`compat32` policy should not be used as a policy for + :class:`~email.message.EmailMessage` objects, and should only be used + to serialize messages that were created using the :const:`compat32` + policy. + .. rubric:: Footnotes diff --git a/Doc/library/email.rst b/Doc/library/email.rst index 66c42e4a5008eef..98b47ffd74096c0 100644 --- a/Doc/library/email.rst +++ b/Doc/library/email.rst @@ -4,18 +4,15 @@ .. module:: email :synopsis: Package supporting the parsing, manipulating, and generating email messages. -.. moduleauthor:: Barry A. Warsaw , - R. David Murray -.. sectionauthor:: R. David Murray **Source code:** :source:`Lib/email/__init__.py` -------------- -The :mod:`email` package is a library for managing email messages. It is +The :mod:`!email` package is a library for managing email messages. It is specifically *not* designed to do any sending of email messages to SMTP (:rfc:`2821`), NNTP, or other servers; those are functions of modules such as -:mod:`smtplib`. The :mod:`email` package attempts to be as +:mod:`smtplib`. The :mod:`!email` package attempts to be as RFC-compliant as possible, supporting :rfc:`5322` and :rfc:`6532`, as well as such MIME-related RFCs as :rfc:`2045`, :rfc:`2046`, :rfc:`2047`, :rfc:`2183`, and :rfc:`2231`. @@ -68,7 +65,7 @@ high level structure in question, and not the details of how those structures are represented. Since MIME content types are used widely in modern internet software (not just email), this will be a familiar concept to many programmers. -The following sections describe the functionality of the :mod:`email` package. +The following sections describe the functionality of the :mod:`!email` package. We start with the :mod:`~email.message` object model, which is the primary interface an application will use, and follow that with the :mod:`~email.parser` and :mod:`~email.generator` components. Then we cover the @@ -102,7 +99,7 @@ compatibility reasons. :class:`~email.message.EmailMessage`/:class:`~email.policy.EmailPolicy` API. -Contents of the :mod:`email` package documentation: +Contents of the :mod:`!email` package documentation: .. toctree:: diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst index 611549604fda15e..e0d2c19a3b0737a 100644 --- a/Doc/library/email.utils.rst +++ b/Doc/library/email.utils.rst @@ -8,7 +8,7 @@ -------------- -There are a couple of useful utilities provided in the :mod:`email.utils` +There are a couple of useful utilities provided in the :mod:`!email.utils` module: .. function:: localtime(dt=None) diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index fa102c4a0801036..e0d77229b11802f 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`ensurepip` package provides support for bootstrapping the ``pip`` +The :mod:`!ensurepip` package provides support for bootstrapping the ``pip`` installer into an existing Python installation or virtual environment. This bootstrapping approach reflects the fact that ``pip`` is an independent project with its own release cycle, and the latest available stable version @@ -30,6 +30,8 @@ when creating a virtual environment) or after explicitly uninstalling needed to bootstrap ``pip`` are included as internal parts of the package. +.. include:: ../includes/optional-module.rst + .. seealso:: :ref:`installing-index` @@ -40,7 +42,9 @@ when creating a virtual environment) or after explicitly uninstalling .. include:: ../includes/wasm-mobile-notavail.rst -Command line interface +.. _ensurepip-cli: + +Command-line interface ---------------------- .. program:: ensurepip @@ -95,7 +99,7 @@ Providing both of the script selection options will trigger an exception. Module API ---------- -:mod:`ensurepip` exposes two functions for programmatic use: +:mod:`!ensurepip` exposes two functions for programmatic use: .. function:: version() diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 2cfc2f4962979f4..559a76bb3963e09 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -4,11 +4,6 @@ .. module:: enum :synopsis: Implementation of an enumeration class. -.. moduleauthor:: Ethan Furman -.. sectionauthor:: Barry Warsaw -.. sectionauthor:: Eli Bendersky -.. sectionauthor:: Ethan Furman - .. versionadded:: 3.4 **Source code:** :source:`Lib/enum.py` @@ -61,7 +56,7 @@ are not normal Python classes. See --------------- -Module Contents +Module contents --------------- :class:`EnumType` @@ -120,30 +115,30 @@ Module Contents :class:`StrEnum` defaults to the lower-cased version of the member name, while other Enums default to 1 and increase from there. - :func:`~enum.property` + :deco:`~enum.property` Allows :class:`Enum` members to have attributes without conflicting with member names. The ``value`` and ``name`` attributes are implemented this way. - :func:`unique` + :deco:`unique` Enum class decorator that ensures only one name is bound to any one value. - :func:`verify` + :deco:`verify` Enum class decorator that checks user-selectable constraints on an enumeration. - :func:`member` + :deco:`member` Make ``obj`` a member. Can be used as a decorator. - :func:`nonmember` + :deco:`nonmember` Do not make ``obj`` a member. Can be used as a decorator. - :func:`global_enum` + :deco:`global_enum` Modify the :class:`str() ` and :func:`repr` of an enum to show its members as belonging to the module instead of its class, @@ -153,6 +148,12 @@ Module Contents Return a list of all power-of-two integers contained in a flag. + :func:`enum.bin` + + Like built-in :func:`bin`, except negative values are represented in + two's complement, and the leading bit always indicates sign + (``0`` implies positive, ``1`` implies negative). + .. versionadded:: 3.6 ``Flag``, ``IntFlag``, ``auto`` .. versionadded:: 3.11 ``StrEnum``, ``EnumCheck``, ``ReprEnum``, ``FlagBoundary``, ``property``, ``member``, ``nonmember``, ``global_enum``, ``show_flag_values`` @@ -160,7 +161,7 @@ Module Contents --------------- -Data Types +Data types ---------- @@ -175,6 +176,10 @@ Data Types final *enum*, as well as creating the enum members, properly handling duplicates, providing iteration over the enum class, etc. + .. versionadded:: 3.11 + + Before 3.11 ``EnumType`` was called ``EnumMeta``, which is still available as an alias. + .. method:: EnumType.__call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) This method is called in two different ways: @@ -206,7 +211,7 @@ Data Types >>> Color.RED.value in Color True - .. versionchanged:: 3.12 + .. versionchanged:: 3.12 Before Python 3.12, a ``TypeError`` is raised if a non-Enum-member is used in a containment check. @@ -217,7 +222,7 @@ Data Types names of the members in *cls*:: >>> dir(Color) - ['BLUE', 'GREEN', 'RED', '__class__', '__contains__', '__doc__', '__getitem__', '__init_subclass__', '__iter__', '__len__', '__members__', '__module__', '__name__', '__qualname__'] + ['BLUE', 'GREEN', 'RED', '__class__', '__contains__', '__doc__', '__getitem__', '__init_subclass__', '__iter__', '__len__', '__members__', '__module__', '__name__', '__qualname__', '_generate_next_value_', '_missing_'] .. method:: EnumType.__getitem__(cls, name) @@ -235,7 +240,7 @@ Data Types .. method:: EnumType.__len__(cls) - Returns the number of member in *cls*:: + Returns the number of members in *cls*:: >>> len(Color) 3 @@ -251,20 +256,6 @@ Data Types >>> list(reversed(Color)) [, , ] - .. method:: EnumType._add_alias_ - - Adds a new name as an alias to an existing member. Raises a - :exc:`NameError` if the name is already assigned to a different member. - - .. method:: EnumType._add_value_alias_ - - Adds a new value as an alias to an existing member. Raises a - :exc:`ValueError` if the value is already linked with a different member. - - .. versionadded:: 3.11 - - Before 3.11 ``EnumType`` was called ``EnumMeta``, which is still available as an alias. - .. class:: Enum @@ -311,6 +302,28 @@ Data Types No longer used, kept for backward compatibility. (class attribute, removed during class creation). + The :attr:`~Enum._order_` attribute can be provided to help keep Python 2 / Python 3 code in sync. + It will be checked against the actual order of the enumeration and raise an error if the two do not match:: + + >>> class Color(Enum): + ... _order_ = 'RED GREEN BLUE' + ... RED = 1 + ... BLUE = 3 + ... GREEN = 2 + ... + Traceback (most recent call last): + ... + TypeError: member order does not match _order_: + ['RED', 'BLUE', 'GREEN'] + ['RED', 'GREEN', 'BLUE'] + + .. note:: + + In Python 2 code the :attr:`~Enum._order_` attribute is necessary as definition + order is lost before it can be recorded. + + .. versionadded:: 3.6 + .. attribute:: Enum._ignore_ ``_ignore_`` is only used during creation and is removed from the @@ -320,12 +333,15 @@ Data Types names will also be removed from the completed enumeration. See :ref:`TimePeriod ` for an example. + .. versionadded:: 3.7 + .. method:: Enum.__dir__(self) Returns ``['__class__', '__doc__', '__module__', 'name', 'value']`` and any public methods defined on *self.__class__*:: - >>> from datetime import date + >>> from enum import Enum + >>> import datetime as dt >>> class Weekday(Enum): ... MONDAY = 1 ... TUESDAY = 2 @@ -336,10 +352,10 @@ Data Types ... SUNDAY = 7 ... @classmethod ... def today(cls): - ... print('today is %s' % cls(date.today().isoweekday()).name) + ... print(f'today is {cls(dt.date.today().isoweekday()).name}') ... >>> dir(Weekday.SATURDAY) - ['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'today', 'value'] + ['__class__', '__doc__', '__eq__', '__hash__', '__module__', '_add_alias_', '_add_value_alias_', '_generate_next_value_', '_missing_', 'name', 'today', 'value'] .. method:: Enum._generate_next_value_(name, start, count, last_values) @@ -349,9 +365,18 @@ Data Types :last_values: A list of the previous values. A *staticmethod* that is used to determine the next value returned by - :class:`auto`:: + :class:`auto`. - >>> from enum import auto + .. note:: + For standard :class:`Enum` classes the next value chosen is the highest + value seen incremented by one. + + For :class:`Flag` classes the next value chosen will be the next highest + power-of-two. + + This method may be overridden, e.g.:: + + >>> from enum import auto, Enum >>> class PowersOfThree(Enum): ... @staticmethod ... def _generate_next_value_(name, start, count, last_values): @@ -362,6 +387,10 @@ Data Types >>> PowersOfThree.SECOND.value 9 + .. versionadded:: 3.6 + .. versionchanged:: 3.13 + Prior versions would use the last seen value instead of the highest value. + .. method:: Enum.__init__(self, *args, **kwds) By default, does nothing. If multiple values are given in the member @@ -383,7 +412,7 @@ Data Types A *classmethod* for looking up values not found in *cls*. By default it does nothing, but can be overridden to implement custom search behavior:: - >>> from enum import StrEnum + >>> from enum import auto, StrEnum >>> class Build(StrEnum): ... DEBUG = auto() ... OPTIMIZED = auto() @@ -400,6 +429,8 @@ Data Types >>> Build('deBUG') + .. versionadded:: 3.6 + .. method:: Enum.__new__(cls, *args, **kwds) By default, doesn't exist. If specified, either in the enum class @@ -422,6 +453,7 @@ Data Types Returns the string used for *repr()* calls. By default, returns the *Enum* name, member name, and value, but can be overridden:: + >>> from enum import auto, Enum >>> class OtherStyle(Enum): ... ALTERNATE = auto() ... OTHER = auto() @@ -438,6 +470,7 @@ Data Types Returns the string used for *str()* calls. By default, returns the *Enum* name and member name, but can be overridden:: + >>> from enum import auto, Enum >>> class OtherStyle(Enum): ... ALTERNATE = auto() ... OTHER = auto() @@ -453,6 +486,7 @@ Data Types Returns the string used for *format()* and *f-string* calls. By default, returns :meth:`__str__` return value, but can be overridden:: + >>> from enum import auto, Enum >>> class OtherStyle(Enum): ... ALTERNATE = auto() ... OTHER = auto() @@ -468,7 +502,32 @@ Data Types Using :class:`auto` with :class:`Enum` results in integers of increasing value, starting with ``1``. - .. versionchanged:: 3.12 Added :ref:`enum-dataclass-support` + .. versionchanged:: 3.12 Added :ref:`enum-dataclass-support`. + + .. method:: Enum._add_alias_ + + Adds a new name as an alias to an existing member:: + + >>> Color.RED._add_alias_("ERROR") + >>> Color.ERROR + + + Raises a :exc:`NameError` if the name is already assigned to a different member. + + .. versionadded:: 3.13 + + .. method:: Enum._add_value_alias_ + + Adds a new value as an alias to an existing member:: + + >>> Color.RED._add_value_alias_(42) + >>> Color(42) + + + | Raises a :exc:`ValueError` if the value is already linked with a different member. + | See :ref:`multi-value-enum` for an example. + + .. versionadded:: 3.13 .. class:: IntEnum @@ -570,7 +629,7 @@ Data Types >>> white in purple False - .. method:: __iter__(self): + .. method:: __iter__(self) Returns all contained non-alias members:: @@ -581,7 +640,7 @@ Data Types .. versionadded:: 3.11 - .. method:: __len__(self): + .. method:: __len__(self) Returns number of members in flag:: @@ -592,7 +651,7 @@ Data Types .. versionadded:: 3.11 - .. method:: __bool__(self): + .. method:: __bool__(self) Returns *True* if any members in flag, *False* otherwise:: @@ -629,7 +688,7 @@ Data Types >>> purple ^ Color.GREEN - .. method:: __invert__(self): + .. method:: __invert__(self) Returns all the flags in *type(self)* that are not in *self*:: @@ -865,6 +924,8 @@ Data Types --------------- +.. _enum-dunder-sunder: + Supported ``__dunder__`` names """""""""""""""""""""""""""""" @@ -872,17 +933,13 @@ Supported ``__dunder__`` names items. It is only available on the class. :meth:`~Enum.__new__`, if specified, must create and return the enum members; -it is also a very good idea to set the member's :attr:`!_value_` appropriately. +it is also a very good idea to set the member's :attr:`~Enum._value_` appropriately. Once all the members are created it is no longer used. Supported ``_sunder_`` names """""""""""""""""""""""""""" -- :meth:`~EnumType._add_alias_` -- adds a new name as an alias to an existing - member. -- :meth:`~EnumType._add_value_alias_` -- adds a new value as an alias to an - existing member. - :attr:`~Enum._name_` -- name of the member - :attr:`~Enum._value_` -- value of the member; can be set in ``__new__`` - :meth:`~Enum._missing_` -- a lookup function used when a value is not found; @@ -892,16 +949,14 @@ Supported ``_sunder_`` names from the final class - :attr:`~Enum._order_` -- no longer used, kept for backward compatibility (class attribute, removed during class creation) + - :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for an enum member; may be overridden - .. note:: - - For standard :class:`Enum` classes the next value chosen is the highest - value seen incremented by one. - - For :class:`Flag` classes the next value chosen will be the next highest - power-of-two. +- :meth:`~Enum._add_alias_` -- adds a new name as an alias to an existing + member. +- :meth:`~Enum._add_value_alias_` -- adds a new value as an alias to an + existing member. - While ``_sunder_`` names are generally reserved for the further development of the :class:`Enum` class and can not be used, some are explicitly allowed: @@ -915,26 +970,27 @@ Supported ``_sunder_`` names --------------- -Utilities and Decorators +Utilities and decorators ------------------------ .. class:: auto *auto* can be used in place of a value. If used, the *Enum* machinery will call an :class:`Enum`'s :meth:`~Enum._generate_next_value_` to get an appropriate value. - For :class:`Enum` and :class:`IntEnum` that appropriate value will be the last value plus - one; for :class:`Flag` and :class:`IntFlag` it will be the first power-of-two greater - than the highest value; for :class:`StrEnum` it will be the lower-cased version of + For :class:`Enum` and :class:`IntEnum` that appropriate value will be the highest value seen + plus one; for :class:`Flag` and :class:`IntFlag` it will be the first power-of-two greater + than the highest value seen; for :class:`StrEnum` it will be the lower-cased version of the member's name. Care must be taken if mixing *auto()* with manually specified values. - *auto* instances are only resolved when at the top level of an assignment: + *auto* instances are only resolved when at the top level of an assignment, either by + itself or as part of a tuple: * ``FIRST = auto()`` will work (auto() is replaced with ``1``); * ``SECOND = auto(), -2`` will work (auto is replaced with ``2``, so ``2, -2`` is used to create the ``SECOND`` enum member; - * ``THREE = [auto(), -3]`` will *not* work (``, -3`` is used to - create the ``THREE`` enum member) + * ``THIRD = [auto(), -3]`` will *not* work (``[, -3]`` is used to + create the ``THIRD`` enum member) .. versionchanged:: 3.11.1 @@ -944,17 +1000,17 @@ Utilities and Decorators ``_generate_next_value_`` can be overridden to customize the values used by *auto*. - .. note:: in 3.13 the default ``_generate_next_value_`` will always return + .. note:: In version 3.13 the default ``_generate_next_value_`` will always return the highest member value incremented by 1, and will fail if any member is an incompatible type. .. decorator:: property - A decorator similar to the built-in *property*, but specifically for + A decorator similar to the built-in :deco:`property`, but specifically for enumerations. It allows member attributes to have the same names as members themselves. - .. note:: the *property* and the member must be defined in separate classes; + .. note:: The *property* and the member must be defined in separate classes; for example, the *value* and *name* attributes are defined in the *Enum* class, and *Enum* subclasses can define members with the names ``value`` and ``name``. @@ -1015,6 +1071,20 @@ Utilities and Decorators .. versionadded:: 3.11 +.. function:: bin(num, max_bits=None) + + Like built-in :func:`bin`, except negative values are represented in + two's complement, and the leading bit always indicates sign + (``0`` implies positive, ``1`` implies negative). + + >>> import enum + >>> enum.bin(10) + '0b0 1010' + >>> enum.bin(~10) # ~10 is -11 + '0b1 0101' + + .. versionadded:: 3.11 + --------------- Notes diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index a73cc2e8a2dd3e5..3775d5ac81a2736 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -221,7 +221,7 @@ The following exceptions are the exceptions that are usually raised. .. exception:: EOFError Raised when the :func:`input` function hits an end-of-file condition (EOF) - without reading any data. (Note: the :meth:`!io.IOBase.read` and + without reading any data. (Note: the :meth:`io.TextIOBase.read` and :meth:`io.IOBase.readline` methods return an empty string when they hit EOF.) @@ -266,6 +266,12 @@ The following exceptions are the exceptions that are usually raised. .. versionadded:: 3.6 +.. exception:: ImportCycleError + + A subclass of :exc:`ImportError` which is raised when a lazy import fails + because it (directly or indirectly) tries to import itself. + + .. versionadded:: 3.15 .. exception:: IndexError @@ -450,7 +456,7 @@ The following exceptions are the exceptions that are usually raised. :meth:`threading.Thread.join` can now raise this exception. - .. versionchanged:: next + .. versionchanged:: 3.15 This exception may be raised when acquiring :meth:`threading.Lock` or :meth:`threading.RLock`. @@ -742,8 +748,8 @@ depending on the system error code. .. attribute:: characters_written - An integer containing the number of characters written to the stream - before it blocked. This attribute is available when using the + An integer containing the number of **bytes** written to the stream + before it blocked. This attribute is available when using the buffered I/O classes from the :mod:`io` module. .. exception:: ChildProcessError @@ -897,6 +903,9 @@ The following exceptions are used as warning categories; see the Base class for warnings about dubious syntax. + This warning is typically emitted when compiling Python source code, and usually won't be reported + when running already compiled code. + .. exception:: RuntimeWarning @@ -975,6 +984,15 @@ their subgroups based on the types of the contained exceptions. raises a :exc:`TypeError` if any contained exception is not an :exc:`Exception` subclass. + Exception groups are :ref:`generic ` over the type of their + contained exceptions. + + .. impl-detail:: + + The ``excs`` parameter may be any sequence, but lists and tuples are + specifically processed more efficiently here. For optimal performance, + pass a tuple as ``excs``. + .. attribute:: message The ``msg`` argument to the constructor. This is a read-only attribute. diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index 677966a8b2eaabb..905e1acd433617d 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -31,7 +31,8 @@ tracebacks: * Each string is limited to 500 characters. * Only the filename, the function name and the line number are displayed. (no source code) -* It is limited to 100 frames and 100 threads. +* It is limited to 100 frames per thread, and 100 threads + (configurable via *max_threads*). * The order is reversed: the most recent call is shown first. By default, the Python traceback is written to :data:`sys.stderr`. To see @@ -55,16 +56,20 @@ at Python startup. Dumping the traceback --------------------- -.. function:: dump_traceback(file=sys.stderr, all_threads=True) +.. function:: dump_traceback(file=sys.stderr, all_threads=True, *, max_threads=100) Dump the tracebacks of all threads into *file*. If *all_threads* is - ``False``, dump only the current thread. + ``False``, dump only the current thread. *max_threads* caps the number + of threads dumped. .. seealso:: :func:`traceback.print_tb`, which can be used to print a traceback object. .. versionchanged:: 3.5 Added support for passing file descriptor to this function. + .. versionchanged:: 3.15 + Added the *max_threads* keyword argument. + Dumping the C stack ------------------- @@ -100,7 +105,7 @@ instead of the stack, even if the operating system supports dumping stacks. Fault handler state ------------------- -.. function:: enable(file=sys.stderr, all_threads=True, c_stack=True) +.. function:: enable(file=sys.stderr, all_threads=True, c_stack=True, *, max_threads=100) Enable the fault handler: install handlers for the :const:`~signal.SIGSEGV`, :const:`~signal.SIGFPE`, :const:`~signal.SIGABRT`, :const:`~signal.SIGBUS` @@ -116,6 +121,8 @@ Fault handler state traceback, unless the system does not support it. See :func:`dump_c_stack` for more information on compatibility. + *max_threads* caps the number of threads dumped when a fatal signal fires. + .. versionchanged:: 3.5 Added support for passing file descriptor to this function. @@ -133,6 +140,9 @@ Fault handler state .. versionchanged:: 3.14 The dump now displays the C stack trace if *c_stack* is true. + .. versionchanged:: 3.15 + Added the *max_threads* keyword argument. + .. function:: disable() Disable the fault handler: uninstall the signal handlers installed by @@ -146,7 +156,7 @@ Fault handler state Dumping the tracebacks after a timeout -------------------------------------- -.. function:: dump_traceback_later(timeout, repeat=False, file=sys.stderr, exit=False) +.. function:: dump_traceback_later(timeout, repeat=False, file=sys.stderr, exit=False, *, max_threads=100) Dump the tracebacks of all threads, after a timeout of *timeout* seconds, or every *timeout* seconds if *repeat* is ``True``. If *exit* is ``True``, call @@ -154,7 +164,7 @@ Dumping the tracebacks after a timeout :c:func:`!_exit` exits the process immediately, which means it doesn't do any cleanup like flushing file buffers.) If the function is called twice, the new call replaces previous parameters and resets the timeout. The timer has a - sub-second resolution. + sub-second resolution. *max_threads* caps the number of threads dumped. The *file* must be kept open until the traceback is dumped or :func:`cancel_dump_traceback_later` is called: see :ref:`issue with file @@ -168,6 +178,9 @@ Dumping the tracebacks after a timeout .. versionchanged:: 3.7 This function is now always available. + .. versionchanged:: 3.15 + Added the *max_threads* keyword argument. + .. function:: cancel_dump_traceback_later() Cancel the last call to :func:`dump_traceback_later`. @@ -176,11 +189,12 @@ Dumping the tracebacks after a timeout Dumping the traceback on a user signal -------------------------------------- -.. function:: register(signum, file=sys.stderr, all_threads=True, chain=False) +.. function:: register(signum, file=sys.stderr, all_threads=True, chain=False, *, max_threads=100) Register a user signal: install a handler for the *signum* signal to dump the traceback of all threads, or of the current thread if *all_threads* is ``False``, into *file*. Call the previous handler if chain is ``True``. + *max_threads* caps the number of threads dumped. The *file* must be kept open until the signal is unregistered by :func:`unregister`: see :ref:`issue with file descriptors `. @@ -190,6 +204,9 @@ Dumping the traceback on a user signal .. versionchanged:: 3.5 Added support for passing file descriptor to this function. + .. versionchanged:: 3.15 + Added the *max_threads* keyword argument. + .. function:: unregister(signum) Unregister a user signal: uninstall the handler of the *signum* signal diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index 5c078df44ffa8ba..c28e4d6c0cc80cc 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -2,11 +2,8 @@ ========================================================== .. module:: fcntl - :platform: Unix :synopsis: The fcntl() and ioctl() system calls. -.. sectionauthor:: Jaap Vermeulen - .. index:: pair: UNIX; file control pair: UNIX; I/O control @@ -53,7 +50,7 @@ descriptor. the latter setting ``FD_CLOEXEC`` flag in addition. .. versionchanged:: 3.12 - On Linux >= 4.5, the :mod:`fcntl` module exposes the ``FICLONE`` and + On Linux >= 4.5, the :mod:`!fcntl` module exposes the ``FICLONE`` and ``FICLONERANGE`` constants, which allow to share some data of one file with another file by reflinking on some filesystems (e.g., btrfs, OCFS2, and XFS). This behavior is commonly referred to as "copy-on-write". @@ -91,7 +88,7 @@ The module defines the following functions: Perform the operation *cmd* on file descriptor *fd* (file objects providing a :meth:`~io.IOBase.fileno` method are accepted as well). The values used for *cmd* are operating system dependent, and are available as constants - in the :mod:`fcntl` module, using the same names as used in the relevant C + in the :mod:`!fcntl` module, using the same names as used in the relevant C header files. The argument *arg* can either be an integer value, a :term:`bytes-like object`, or a string. The type and size of *arg* must match the type and size of @@ -125,7 +122,7 @@ The module defines the following functions: Add support of arbitrary :term:`bytes-like objects `, not only :class:`bytes`. - .. versionchanged:: next + .. versionchanged:: 3.15 The size of bytes-like objects is no longer limited to 1024 bytes. @@ -187,7 +184,7 @@ The module defines the following functions: The GIL is always released during a system call. System calls failing with EINTR are automatically retried. - .. versionchanged:: next + .. versionchanged:: 3.15 The size of not mutated bytes-like objects is no longer limited to 1024 bytes. diff --git a/Doc/library/filecmp.rst b/Doc/library/filecmp.rst index abd1b8c826d1705..f8365b44c5a502d 100644 --- a/Doc/library/filecmp.rst +++ b/Doc/library/filecmp.rst @@ -4,17 +4,15 @@ .. module:: filecmp :synopsis: Compare files efficiently. -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/filecmp.py` -------------- -The :mod:`filecmp` module defines functions to compare files and directories, +The :mod:`!filecmp` module defines functions to compare files and directories, with various optional time/correctness trade-offs. For comparing files, see also the :mod:`difflib` module. -The :mod:`filecmp` module defines the following functions: +The :mod:`!filecmp` module defines the following functions: .. function:: cmp(f1, f2, shallow=True) diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index 8f32b11e5653652..5be16797be908cb 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -4,9 +4,6 @@ .. module:: fileinput :synopsis: Loop over standard input or a list of files. -.. moduleauthor:: Guido van Rossum -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/fileinput.py` -------------- diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index ee654b7a83e203b..fc99b7fd1e7523a 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -51,7 +51,7 @@ Unless stated otherwise, "filename string" and "pattern string" either refer to functions documented below do not allow to mix a :class:`!bytes` pattern with a :class:`!str` filename, and vice-versa. -Finally, note that :func:`functools.lru_cache` with a *maxsize* of 32768 +Finally, note that :deco:`functools.lru_cache` with a *maxsize* of 32768 is used to cache the (typed) compiled regex patterns in the following functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`, :func:`.filterfalse`. @@ -103,7 +103,8 @@ functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`, :func:`.filter .. function:: translate(pat) Return the shell-style pattern *pat* converted to a regular expression for - using with :func:`re.match`. The pattern is expected to be a :class:`str`. + using with :func:`re.prefixmatch`. The pattern is expected to be a + :class:`str`. Example: @@ -113,7 +114,7 @@ functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`, :func:`.filter >>> regex '(?s:.*\\.txt)\\z' >>> reobj = re.compile(regex) - >>> reobj.match('foobar.txt') + >>> reobj.prefixmatch('foobar.txt') diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index 8796056b4b8722c..b02e7b5b641136d 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -4,18 +4,15 @@ .. module:: fractions :synopsis: Rational numbers. -.. moduleauthor:: Jeffrey Yasskin -.. sectionauthor:: Jeffrey Yasskin - **Source code:** :source:`Lib/fractions.py` -------------- -The :mod:`fractions` module provides support for rational number arithmetic. +The :mod:`!fractions` module provides support for rational number arithmetic. -A Fraction instance can be constructed from a pair of integers, from -another rational number, or from a string. +A Fraction instance can be constructed from a pair of rational numbers, from +a single number, or from a string. .. index:: single: as_integer_ratio() @@ -25,8 +22,8 @@ another rational number, or from a string. The first version requires that *numerator* and *denominator* are instances of :class:`numbers.Rational` and returns a new :class:`Fraction` instance - with value equal to ``numerator/denominator`` where the denominator is positive. - If *denominator* is ``0``, it raises a :exc:`ZeroDivisionError`. + with a value equal to ``numerator/denominator``. + If *denominator* is zero, it raises a :exc:`ZeroDivisionError`. The second version requires that *number* is an instance of :class:`numbers.Rational` or has the :meth:`!as_integer_ratio` method @@ -125,7 +122,8 @@ another rational number, or from a string. .. attribute:: denominator - Denominator of the Fraction in lowest term. + Denominator of the Fraction in lowest terms. + Guaranteed to be positive. .. method:: as_integer_ratio() diff --git a/Doc/library/frameworks.rst b/Doc/library/frameworks.rst index 15ceeec9c255edb..f8e2f6bb18cb1cb 100644 --- a/Doc/library/frameworks.rst +++ b/Doc/library/frameworks.rst @@ -1,18 +1,13 @@ +:orphan: + .. _frameworks: ****************** -Program Frameworks +Program frameworks ****************** -The modules described in this chapter are frameworks that will largely dictate -the structure of your program. Currently the modules described here are all -oriented toward writing command-line interfaces. - -The full list of modules described in this chapter is: - - -.. toctree:: +This chapter is no longer maintained, and the modules it contained have been moved to their respective topical documentation. - turtle.rst - cmd.rst - shlex.rst +* :mod:`cmd` — :doc:`Command Line Interface Libraries <./cmdlinelibs>` +* :mod:`shlex` — :doc:`Unix Specific Services <./unix>` +* :mod:`turtle` — :doc:`Graphical User Interfaces with Tk <./tk>` diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index bb15322067245e8..e1baeff3f373bf1 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -23,7 +23,7 @@ The default encoding is UTF-8, following :rfc:`2640`. .. include:: ../includes/wasm-notavail.rst -Here's a sample session using the :mod:`ftplib` module:: +Here's a sample session using the :mod:`!ftplib` module:: >>> from ftplib import FTP >>> ftp = FTP('ftp.us.debian.org') # connect to host, default port @@ -524,14 +524,9 @@ FTP_TLS objects :class:`!FTP_TLS` class inherits from :class:`FTP`, defining these additional methods and attributes: - .. attribute:: FTP_TLS.ssl_version - - The SSL version to use (defaults to :data:`ssl.PROTOCOL_SSLv23`). - .. method:: FTP_TLS.auth() - Set up a secure control connection by using TLS or SSL, depending on what - is specified in the :attr:`ssl_version` attribute. + Set up a secure control connection by using TLS. .. versionchanged:: 3.4 The method now supports hostname check with @@ -548,7 +543,7 @@ FTP_TLS objects .. method:: FTP_TLS.prot_p() - Set up secure data connection. + Set up secure data connection by using TLS. .. method:: FTP_TLS.prot_c() diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 80bd1275973f8d1..a4d37beb436899a 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -19,24 +19,25 @@ are always available. They are listed here in alphabetical order. | | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | | | | | :func:`float` | | :func:`max` | | |func-set|_ | | | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | -| | :func:`bin` | | |func-frozenset|_ | | :func:`min` | | :func:`slice` | -| | :func:`bool` | | | | | | :func:`sorted` | -| | :func:`breakpoint` | | **G** | | **N** | | :func:`staticmethod` | -| | |func-bytearray|_ | | :func:`getattr` | | :func:`next` | | |func-str|_ | -| | |func-bytes|_ | | :func:`globals` | | | | :func:`sum` | -| | | | | | **O** | | :func:`super` | -| | **C** | | **H** | | :func:`object` | | | -| | :func:`callable` | | :func:`hasattr` | | :func:`oct` | | **T** | -| | :func:`chr` | | :func:`hash` | | :func:`open` | | |func-tuple|_ | -| | :func:`classmethod` | | :func:`help` | | :func:`ord` | | :func:`type` | -| | :func:`compile` | | :func:`hex` | | | | | -| | :func:`complex` | | | | **P** | | **V** | -| | | | **I** | | :func:`pow` | | :func:`vars` | -| | **D** | | :func:`id` | | :func:`print` | | | -| | :func:`delattr` | | :func:`input` | | :func:`property` | | **Z** | -| | |func-dict|_ | | :func:`int` | | | | :func:`zip` | -| | :func:`dir` | | :func:`isinstance` | | | | | -| | :func:`divmod` | | :func:`issubclass` | | | | **_** | +| | :func:`bin` | | |func-frozendict|_ | | :func:`min` | | :func:`sentinel` | +| | :func:`bool` | | |func-frozenset|_ | | | | :func:`slice` | +| | :func:`breakpoint` | | | | **N** | | :func:`sorted` | +| | |func-bytearray|_ | | **G** | | :func:`next` | | :func:`staticmethod` | +| | |func-bytes|_ | | :func:`getattr` | | | | |func-str|_ | +| | | | :func:`globals` | | **O** | | :func:`sum` | +| | **C** | | | | :func:`object` | | :func:`super` | +| | :func:`callable` | | **H** | | :func:`oct` | | | +| | :func:`chr` | | :func:`hasattr` | | :func:`open` | | **T** | +| | :func:`classmethod` | | :func:`hash` | | :func:`ord` | | |func-tuple|_ | +| | :func:`compile` | | :func:`help` | | | | :func:`type` | +| | :func:`complex` | | :func:`hex` | | **P** | | | +| | | | | | :func:`pow` | | **V** | +| | **D** | | **I** | | :func:`print` | | :func:`vars` | +| | :func:`delattr` | | :func:`id` | | :func:`property` | | | +| | |func-dict|_ | | :func:`input` | | | | **Z** | +| | :func:`dir` | | :func:`int` | | | | :func:`zip` | +| | :func:`divmod` | | :func:`isinstance` | | | | | +| | | | :func:`issubclass` | | | | **_** | | | | | :func:`iter` | | | | :func:`__import__` | +-------------------------+-----------------------+-----------------------+-------------------------+ @@ -44,6 +45,7 @@ are always available. They are listed here in alphabetical order. used, with replacement texts to make the output in the table consistent .. |func-dict| replace:: ``dict()`` +.. |func-frozendict| replace:: ``frozendict()`` .. |func-frozenset| replace:: ``frozenset()`` .. |func-memoryview| replace:: ``memoryview()`` .. |func-set| replace:: ``set()`` @@ -54,7 +56,7 @@ are always available. They are listed here in alphabetical order. .. |func-bytearray| replace:: ``bytearray()`` .. |func-bytes| replace:: ``bytes()`` -.. function:: abs(x) +.. function:: abs(number, /) Return the absolute value of a number. The argument may be an integer, a floating-point number, or an object implementing @@ -62,7 +64,7 @@ are always available. They are listed here in alphabetical order. If the argument is a complex number, its magnitude is returned. -.. function:: aiter(async_iterable) +.. function:: aiter(async_iterable, /) Return an :term:`asynchronous iterator` for an :term:`asynchronous iterable`. Equivalent to calling ``x.__aiter__()``. @@ -71,7 +73,7 @@ are always available. They are listed here in alphabetical order. .. versionadded:: 3.10 -.. function:: all(iterable) +.. function:: all(iterable, /) Return ``True`` if all elements of the *iterable* are true (or if the iterable is empty). Equivalent to:: @@ -83,8 +85,8 @@ are always available. They are listed here in alphabetical order. return True -.. awaitablefunction:: anext(async_iterator) - anext(async_iterator, default) +.. awaitablefunction:: anext(async_iterator, /) + anext(async_iterator, default, /) When awaited, return the next item from the given :term:`asynchronous iterator`, or *default* if given and the iterator is exhausted. @@ -99,7 +101,7 @@ are always available. They are listed here in alphabetical order. .. versionadded:: 3.10 -.. function:: any(iterable) +.. function:: any(iterable, /) Return ``True`` if any element of the *iterable* is true. If the iterable is empty, return ``False``. Equivalent to:: @@ -111,7 +113,7 @@ are always available. They are listed here in alphabetical order. return False -.. function:: ascii(object) +.. function:: ascii(object, /) As :func:`repr`, return a string containing a printable representation of an object, but escape the non-ASCII characters in the string returned by @@ -119,10 +121,10 @@ are always available. They are listed here in alphabetical order. similar to that returned by :func:`repr` in Python 2. -.. function:: bin(x) +.. function:: bin(integer, /) Convert an integer number to a binary string prefixed with "0b". The result - is a valid Python expression. If *x* is not a Python :class:`int` object, it + is a valid Python expression. If *integer* is not a Python :class:`int` object, it has to define an :meth:`~object.__index__` method that returns an integer. Some examples: @@ -138,6 +140,8 @@ are always available. They are listed here in alphabetical order. >>> f'{14:#b}', f'{14:b}' ('0b1110', '1110') + See also :func:`enum.bin` to represent negative values as twos-complement. + See also :func:`format` for more information. @@ -183,8 +187,7 @@ are always available. They are listed here in alphabetical order. .. _func-bytearray: .. class:: bytearray(source=b'') - bytearray(source, encoding) - bytearray(source, encoding, errors) + bytearray(source, encoding, errors='strict') :noindex: Return a new array of bytes. The :class:`bytearray` class is a mutable @@ -215,8 +218,7 @@ are always available. They are listed here in alphabetical order. .. _func-bytes: .. class:: bytes(source=b'') - bytes(source, encoding) - bytes(source, encoding, errors) + bytes(source, encoding, errors='strict') :noindex: Return a new "bytes" object which is an immutable sequence of integers in @@ -231,7 +233,7 @@ are always available. They are listed here in alphabetical order. See also :ref:`binaryseq`, :ref:`typebytes`, and :ref:`bytes-methods`. -.. function:: callable(object) +.. function:: callable(object, /) Return :const:`True` if the *object* argument appears callable, :const:`False` if not. If this returns ``True``, it is still possible that a @@ -244,14 +246,14 @@ are always available. They are listed here in alphabetical order. in Python 3.2. -.. function:: chr(i) +.. function:: chr(codepoint, /) - Return the string representing a character whose Unicode code point is the - integer *i*. For example, ``chr(97)`` returns the string ``'a'``, while + Return the string representing a character with the specified Unicode code point. + For example, ``chr(97)`` returns the string ``'a'``, while ``chr(8364)`` returns the string ``'€'``. This is the inverse of :func:`ord`. The valid range for the argument is from 0 through 1,114,111 (0x10FFFF in - base 16). :exc:`ValueError` will be raised if *i* is outside that range. + base 16). :exc:`ValueError` will be raised if it is outside that range. .. decorator:: classmethod @@ -294,7 +296,9 @@ are always available. They are listed here in alphabetical order. :func:`property`. -.. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1) +.. function:: compile(source, filename, mode, flags=0, \ + dont_inherit=False, optimize=-1, \ + *, module=None) Compile the *source* into a code or AST object. Code objects can be executed by :func:`exec` or :func:`eval`. *source* can either be a normal string, a @@ -336,8 +340,12 @@ are always available. They are listed here in alphabetical order. ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false) or ``2`` (docstrings are removed too). - This function raises :exc:`SyntaxError` if the compiled source is invalid, - and :exc:`ValueError` if the source contains null bytes. + The optional argument *module* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax warnings + by module name. + + This function raises :exc:`SyntaxError` or :exc:`ValueError` if the compiled + source is invalid. If you want to parse Python code into its AST representation, see :func:`ast.parse`. @@ -373,6 +381,9 @@ are always available. They are listed here in alphabetical order. ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable support for top-level ``await``, ``async for``, and ``async with``. + .. versionadded:: 3.15 + Added the *module* parameter. + .. class:: complex(number=0, /) complex(string, /) @@ -458,7 +469,7 @@ are always available. They are listed here in alphabetical order. deprecated; it should only be passed as a single positional argument. -.. function:: delattr(object, name) +.. function:: delattr(object, name, /) This is a relative of :func:`setattr`. The arguments are an object and a string. The string must be the name of one of the object's attributes. The @@ -468,20 +479,20 @@ are always available. They are listed here in alphabetical order. .. _func-dict: -.. class:: dict(**kwarg) - dict(mapping, **kwarg) - dict(iterable, **kwarg) +.. class:: dict(**kwargs) + dict(mapping, /, **kwargs) + dict(iterable, /, **kwargs) :noindex: Create a new dictionary. The :class:`dict` object is the dictionary class. - See :class:`dict` and :ref:`typesmapping` for documentation about this class. + See also :ref:`typesmapping` for documentation about this class. - For other containers see the built-in :class:`list`, :class:`set`, and - :class:`tuple` classes, as well as the :mod:`collections` module. + For other containers see the built-in :class:`frozendict`, :class:`list`, + :class:`set`, and :class:`tuple` classes, as well as the :mod:`collections` module. .. function:: dir() - dir(object) + dir(object, /) Without arguments, return the list of names in the current local scope. With an argument, attempt to return a list of valid attributes for that object. @@ -519,7 +530,7 @@ are always available. They are listed here in alphabetical order. >>> dir() # show the names in the module namespace # doctest: +SKIP ['__builtins__', '__name__', 'struct'] >>> dir(struct) # show the names in the struct module # doctest: +SKIP - ['Struct', '__all__', '__builtins__', '__cached__', '__doc__', '__file__', + ['Struct', '__all__', '__builtins__', '__doc__', '__file__', '__initializing__', '__loader__', '__name__', '__package__', '_clearcache', 'calcsize', 'error', 'pack', 'pack_into', 'unpack', 'unpack_from'] @@ -541,7 +552,7 @@ are always available. They are listed here in alphabetical order. class. -.. function:: divmod(a, b) +.. function:: divmod(a, b, /) Take two (non-complex) numbers as arguments and return a pair of numbers consisting of their quotient and remainder when using integer division. With @@ -585,7 +596,7 @@ are always available. They are listed here in alphabetical order. :param globals: The global namespace (default: ``None``). - :type globals: :class:`dict` | ``None`` + :type globals: :class:`dict` | :class:`frozendict` | ``None`` :param locals: The local namespace (default: ``None``). @@ -597,18 +608,19 @@ are always available. They are listed here in alphabetical order. .. warning:: This function executes arbitrary code. Calling it with - user-supplied input may lead to security vulnerabilities. + untrusted user-supplied input will lead to security vulnerabilities. - The *expression* argument is parsed and evaluated as a Python expression + The *source* argument is parsed and evaluated as a Python expression (technically speaking, a condition list) using the *globals* and *locals* mappings as global and local namespace. If the *globals* dictionary is present and does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module :mod:`builtins` is - inserted under that key before *expression* is parsed. That way you can - control what builtins are available to the executed code by inserting your - own ``__builtins__`` dictionary into *globals* before passing it to - :func:`eval`. If the *locals* mapping is omitted it defaults to the - *globals* dictionary. If both mappings are omitted, the expression is + inserted under that key before *source* is parsed. + Overriding ``__builtins__`` can be used to restrict or change the available + names, but this is **not** a security mechanism: the executed code can + still access all builtins. + If the *locals* mapping is omitted it defaults to the + *globals* dictionary. If both mappings are omitted, the source is executed with the *globals* and *locals* in the environment where :func:`eval` is called. Note, *eval()* will only have access to the :term:`nested scopes ` (non-locals) in the enclosing @@ -634,7 +646,7 @@ are always available. They are listed here in alphabetical order. If the given source is a string, then leading and trailing spaces and tabs are stripped. - See :func:`ast.literal_eval` for a function that can safely evaluate strings + See :func:`ast.literal_eval` for a function to evaluate strings with expressions containing only literals. .. audit-event:: exec code_object eval @@ -651,6 +663,10 @@ are always available. They are listed here in alphabetical order. The semantics of the default *locals* namespace have been adjusted as described for the :func:`locals` builtin. + .. versionchanged:: 3.15 + + *globals* can now be a :class:`frozendict`. + .. index:: pair: built-in function; exec .. function:: exec(source, /, globals=None, locals=None, *, closure=None) @@ -658,7 +674,7 @@ are always available. They are listed here in alphabetical order. .. warning:: This function executes arbitrary code. Calling it with - user-supplied input may lead to security vulnerabilities. + untrusted user-supplied input will lead to security vulnerabilities. This function supports dynamic execution of Python code. *source* must be either a string or a code object. If it is a string, the string is parsed as @@ -689,9 +705,10 @@ are always available. They are listed here in alphabetical order. If the *globals* dictionary does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module - :mod:`builtins` is inserted under that key. That way you can control what - builtins are available to the executed code by inserting your own - ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`. + :mod:`builtins` is inserted under that key. + Overriding ``__builtins__`` can be used to restrict or change the available + names, but this is **not** a security mechanism: the executed code can + still access all builtins. The *closure* argument specifies a closure--a tuple of cellvars. It's only valid when the *object* is a code object containing @@ -728,8 +745,12 @@ are always available. They are listed here in alphabetical order. The semantics of the default *locals* namespace have been adjusted as described for the :func:`locals` builtin. + .. versionchanged:: 3.15 + + *globals* can now be a :class:`frozendict`. + -.. function:: filter(function, iterable) +.. function:: filter(function, iterable, /) Construct an iterator from those elements of *iterable* for which *function* is true. *iterable* may be either a sequence, a container which @@ -823,7 +844,7 @@ are always available. They are listed here in alphabetical order. single: __format__ single: string; format() (built-in function) -.. function:: format(value, format_spec="") +.. function:: format(value, format_spec="", /) Convert a *value* to a "formatted" representation, as controlled by *format_spec*. The interpretation of *format_spec* will depend on the type @@ -845,12 +866,27 @@ are always available. They are listed here in alphabetical order. if *format_spec* is not an empty string. +.. _func-frozendict: +.. class:: frozendict(**kwargs) + frozendict(mapping, /, **kwargs) + frozendict(iterable, /, **kwargs) + :noindex: + + Create a new frozen dictionary. The :class:`frozendict` object is a built-in class. + See also :ref:`typesmapping` for documentation about this class. + + For other containers see the built-in :class:`dict`, :class:`list`, :class:`set`, + and :class:`tuple` classes, as well as the :mod:`collections` module. + + .. versionadded:: 3.15 + + .. _func-frozenset: -.. class:: frozenset(iterable=set()) +.. class:: frozenset(iterable=(), /) :noindex: Return a new :class:`frozenset` object, optionally with elements taken from - *iterable*. ``frozenset`` is a built-in class. See :class:`frozenset` and + *iterable*. :class:`frozenset` is a built-in class. See also :ref:`types-set` for documentation about this class. For other containers see the built-in :class:`set`, :class:`list`, @@ -858,8 +894,8 @@ are always available. They are listed here in alphabetical order. module. -.. function:: getattr(object, name) - getattr(object, name, default) +.. function:: getattr(object, name, /) + getattr(object, name, default, /) Return the value of the named attribute of *object*. *name* must be a string. If the string is the name of one of the object's attributes, the result is the @@ -883,7 +919,7 @@ are always available. They are listed here in alphabetical order. regardless of where the function is called. -.. function:: hasattr(object, name) +.. function:: hasattr(object, name, /) The arguments are an object and a string. The result is ``True`` if the string is the name of one of the object's attributes, ``False`` if not. (This @@ -891,7 +927,7 @@ are always available. They are listed here in alphabetical order. raises an :exc:`AttributeError` or not.) -.. function:: hash(object) +.. function:: hash(object, /) Return the hash value of the object (if it has one). Hash values are integers. They are used to quickly compare dictionary keys during a @@ -926,10 +962,10 @@ are always available. They are listed here in alphabetical order. signatures for callables are now more comprehensive and consistent. -.. function:: hex(x) +.. function:: hex(integer, /) Convert an integer number to a lowercase hexadecimal string prefixed with - "0x". If *x* is not a Python :class:`int` object, it has to define an + "0x". If *integer* is not a Python :class:`int` object, it has to define an :meth:`~object.__index__` method that returns an integer. Some examples: >>> hex(255) @@ -958,7 +994,7 @@ are always available. They are listed here in alphabetical order. :meth:`float.hex` method. -.. function:: id(object) +.. function:: id(object, /) Return the "identity" of an object. This is an integer which is guaranteed to be unique and constant for this object during its lifetime. @@ -971,7 +1007,7 @@ are always available. They are listed here in alphabetical order. .. function:: input() - input(prompt) + input(prompt, /) If the *prompt* argument is present, it is written to standard output without a trailing newline. The function then reads a line from input, converts it @@ -1071,7 +1107,7 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.14 :func:`int` no longer delegates to the :meth:`~object.__trunc__` method. -.. function:: isinstance(object, classinfo) +.. function:: isinstance(object, classinfo, /) Return ``True`` if the *object* argument is an instance of the *classinfo* argument, or of a (direct, indirect, or :term:`virtual `) of *classinfo*. A class is considered a subclass of itself. *classinfo* may be a tuple of class objects (or recursively, other such tuples) - or a :ref:`types-union`, in which case return ``True`` if *class* is a + or a :ref:`types-union`, in which case return ``True`` if *cls* is a subclass of any entry in *classinfo*. In any other case, a :exc:`TypeError` exception is raised. @@ -1102,19 +1138,19 @@ are always available. They are listed here in alphabetical order. *classinfo* can be a :ref:`types-union`. -.. function:: iter(object) - iter(object, sentinel) +.. function:: iter(iterable, /) + iter(callable, sentinel, /) Return an :term:`iterator` object. The first argument is interpreted very differently depending on the presence of the second argument. Without a - second argument, *object* must be a collection object which supports the + second argument, the single argument must be a collection object which supports the :term:`iterable` protocol (the :meth:`~object.__iter__` method), or it must support the sequence protocol (the :meth:`~object.__getitem__` method with integer arguments starting at ``0``). If it does not support either of those protocols, :exc:`TypeError` is raised. If the second argument, *sentinel*, is given, - then *object* must be a callable object. The iterator created in this case - will call *object* with no arguments for each call to its + then the first argument must be a callable object. The iterator created in this case + will call *callable* with no arguments for each call to its :meth:`~iterator.__next__` method; if the value returned is equal to *sentinel*, :exc:`StopIteration` will be raised, otherwise the value will be returned. @@ -1131,7 +1167,7 @@ are always available. They are listed here in alphabetical order. process_block(block) -.. function:: len(s) +.. function:: len(object, /) Return the length (the number of items) of an object. The argument may be a sequence (such as a string, bytes, tuple, list, or range) or a collection @@ -1144,8 +1180,7 @@ are always available. They are listed here in alphabetical order. .. _func-list: -.. class:: list() - list(iterable) +.. class:: list(iterable=(), /) :noindex: Rather than being a function, :class:`list` is actually a mutable @@ -1220,9 +1255,9 @@ are always available. They are listed here in alphabetical order. Added the *strict* parameter. -.. function:: max(iterable, *, key=None) - max(iterable, *, default, key=None) - max(arg1, arg2, *args, key=None) +.. function:: max(iterable, /, *, key=None) + max(iterable, /, *, default, key=None) + max(arg1, arg2, /, *args, key=None) Return the largest item in an iterable or the largest of two or more arguments. @@ -1258,9 +1293,9 @@ are always available. They are listed here in alphabetical order. :ref:`typememoryview` for more information. -.. function:: min(iterable, *, key=None) - min(iterable, *, default, key=None) - min(arg1, arg2, *args, key=None) +.. function:: min(iterable, /, *, key=None) + min(iterable, /, *, default, key=None) + min(arg1, arg2, /, *args, key=None) Return the smallest item in an iterable or the smallest of two or more arguments. @@ -1288,8 +1323,8 @@ are always available. They are listed here in alphabetical order. The *key* can be ``None``. -.. function:: next(iterator) - next(iterator, default) +.. function:: next(iterator, /) + next(iterator, default, /) Retrieve the next item from the :term:`iterator` by calling its :meth:`~iterator.__next__` method. If *default* is given, it is returned @@ -1310,10 +1345,10 @@ are always available. They are listed here in alphabetical order. :class:`object`. -.. function:: oct(x) +.. function:: oct(integer, /) Convert an integer number to an octal string prefixed with "0o". The result - is a valid Python expression. If *x* is not a Python :class:`int` object, it + is a valid Python expression. If *integer* is not a Python :class:`int` object, it has to define an :meth:`~object.__index__` method that returns an integer. For example: @@ -1422,38 +1457,10 @@ are always available. They are listed here in alphabetical order. *errors* is an optional string that specifies how encoding and decoding errors are to be handled—this cannot be used in binary mode. - A variety of standard error handlers are available - (listed under :ref:`error-handlers`), though any - error handling name that has been registered with + A variety of standard error handlers are available, + though any error handling name that has been registered with :func:`codecs.register_error` is also valid. The standard names - include: - - * ``'strict'`` to raise a :exc:`ValueError` exception if there is - an encoding error. The default value of ``None`` has the same - effect. - - * ``'ignore'`` ignores errors. Note that ignoring encoding errors - can lead to data loss. - - * ``'replace'`` causes a replacement marker (such as ``'?'``) to be inserted - where there is malformed data. - - * ``'surrogateescape'`` will represent any incorrect bytes as low - surrogate code units ranging from U+DC80 to U+DCFF. - These surrogate code units will then be turned back into - the same bytes when the ``surrogateescape`` error handler is used - when writing data. This is useful for processing files in an - unknown encoding. - - * ``'xmlcharrefreplace'`` is only supported when writing to a file. - Characters not supported by the encoding are replaced with the - appropriate XML character reference :samp:`&#{nnn};`. - - * ``'backslashreplace'`` replaces malformed data by Python's backslashed - escape sequences. - - * ``'namereplace'`` (also only supported when writing) - replaces unsupported characters with ``\N{...}`` escape sequences. + can be found in :ref:`error-handlers`. .. index:: single: universal newlines; open() built-in function @@ -1562,13 +1569,19 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.11 The ``'U'`` mode has been removed. -.. function:: ord(c) +.. function:: ord(character, /) + + Return the ordinal value of a character. - Given a string representing one Unicode character, return an integer - representing the Unicode code point of that character. For example, + If the argument is a one-character string, return the Unicode code point + of that character. For example, ``ord('a')`` returns the integer ``97`` and ``ord('€')`` (Euro sign) returns ``8364``. This is the inverse of :func:`chr`. + If the argument is a :class:`bytes` or :class:`bytearray` object of + length 1, return its single byte value. + For example, ``ord(b'a')`` returns the integer ``97``. + .. function:: pow(base, exp, mod=None) @@ -1577,7 +1590,7 @@ are always available. They are listed here in alphabetical order. ``pow(base, exp) % mod``). The two-argument form ``pow(base, exp)`` is equivalent to using the power operator: ``base**exp``. - The arguments must have numeric types. With mixed operand types, the + When arguments are builtin numeric types with mixed operand types, the coercion rules for binary arithmetic operators apply. For :class:`int` operands, the result has the same type as the operands (after coercion) unless the second argument is negative; in that case, all arguments are @@ -1667,7 +1680,7 @@ are always available. They are listed here in alphabetical order. If given, *doc* will be the docstring of the property attribute. Otherwise, the property will copy *fget*'s docstring (if it exists). This makes it possible to - create read-only properties easily using :func:`property` as a :term:`decorator`:: + create read-only properties easily using :deco:`property` as a :term:`decorator`:: class Parrot: def __init__(self): @@ -1729,15 +1742,15 @@ are always available. They are listed here in alphabetical order. .. _func-range: -.. class:: range(stop) - range(start, stop, step=1) +.. class:: range(stop, /) + range(start, stop, step=1, /) :noindex: Rather than being a function, :class:`range` is actually an immutable sequence type, as documented in :ref:`typesseq-range` and :ref:`typesseq`. -.. function:: repr(object) +.. function:: repr(object, /) Return a string containing a printable representation of an object. For many types, this function makes an attempt to return a string that would yield an @@ -1758,12 +1771,12 @@ are always available. They are listed here in alphabetical order. self.age = age def __repr__(self): - return f"Person('{self.name}', {self.age})" + return f"Person({self.name!r}, {self.age!r})" -.. function:: reversed(seq) +.. function:: reversed(object, /) - Return a reverse :term:`iterator`. *seq* must be an object which has + Return a reverse :term:`iterator`. The argument must be an object which has a :meth:`~object.__reversed__` method or supports the sequence protocol (the :meth:`~object.__len__` method and the :meth:`~object.__getitem__` method with integer arguments starting at ``0``). @@ -1797,12 +1810,11 @@ are always available. They are listed here in alphabetical order. .. _func-set: -.. class:: set() - set(iterable) +.. class:: set(iterable=(), /) :noindex: Return a new :class:`set` object, optionally with elements taken from - *iterable*. ``set`` is a built-in class. See :class:`set` and + *iterable*. :class:`set` is a built-in class. See also :ref:`types-set` for documentation about this class. For other containers see the built-in :class:`frozenset`, :class:`list`, @@ -1810,7 +1822,7 @@ are always available. They are listed here in alphabetical order. module. -.. function:: setattr(object, name, value) +.. function:: setattr(object, name, value, /) This is the counterpart of :func:`getattr`. The arguments are an object, a string, and an arbitrary value. The string may name an existing attribute or a @@ -1832,26 +1844,89 @@ are always available. They are listed here in alphabetical order. :func:`setattr`. -.. class:: slice(stop) - slice(start, stop, step=None) +.. class:: sentinel(name, /, *, repr=None) + + Return a new unique sentinel object. *name* must be a :class:`str`, and is + used by default as the returned object's representation:: + + >>> MISSING = sentinel("MISSING") + >>> MISSING + MISSING + + The optional *repr* argument can be used to specify a different representation:: + + >>> MISSING = sentinel("MISSING", repr="") + >>> MISSING + + + Sentinel objects are truthy and compare equal only to themselves. They are + intended to be compared with the :keyword:`is` operator. + + ``sentinel`` does not support subclassing. + + Shallow and deep copies of a sentinel object return the object itself. + + Sentinels are conventionally assigned to a variable with a matching name. + Sentinels defined in this way can be used in :term:`type hints `:: + + MISSING = sentinel("MISSING") + + def next_value(default: int | MISSING = MISSING): + ... + + Sentinel objects support the :ref:`| ` operator for use in type expressions. + + :mod:`Pickling ` is supported for sentinel objects that are + placed in the global scope of a module under a name matching the sentinel's + name, and for sentinels placed in class scopes with a name matching the + :term:`qualified name` of the sentinel. Other sentinels, such as those + defined in a function scope, are not picklable. The identity of the sentinel is preserved + after pickling:: + + import pickle + + PICKLABLE = sentinel("PICKLABLE") + + assert pickle.loads(pickle.dumps(PICKLABLE)) is PICKLABLE + + class Cls: + PICKLABLE = sentinel("Cls.PICKLABLE") + + assert pickle.loads(pickle.dumps(Cls.PICKLABLE)) is Cls.PICKLABLE + + Sentinel objects have the following attributes: + + .. attribute:: __name__ + + The sentinel's name. + + .. attribute:: __module__ + + The name of the module where the sentinel was created. This attribute is writable. + + .. versionadded:: 3.15 + + +.. class:: slice(stop, /) + slice(start, stop, step=None, /) Return a :term:`slice` object representing the set of indices specified by ``range(start, stop, step)``. The *start* and *step* arguments default to ``None``. - Slice objects have read-only data attributes :attr:`!start`, - :attr:`!stop`, and :attr:`!step` which merely return the argument - values (or their default). They have no other explicit functionality; - however, they are used by NumPy and other third-party packages. + Slice objects are also generated when :ref:`slicing syntax ` + is used. For example: ``a[start:stop:step]`` or ``a[start:stop, i]``. + + See :func:`itertools.islice` for an alternate version that returns an + :term:`iterator`. .. attribute:: slice.start - .. attribute:: slice.stop - .. attribute:: slice.step + slice.stop + slice.step - Slice objects are also generated when extended indexing syntax is used. For - example: ``a[start:stop:step]`` or ``a[start:stop, i]``. See - :func:`itertools.islice` for an alternate version that returns an - :term:`iterator`. + These read-only attributes are set to the argument values + (or their default). They have no other explicit functionality; + however, they are used by NumPy and other third-party packages. .. versionchanged:: 3.12 Slice objects are now :term:`hashable` (provided :attr:`~slice.start`, @@ -1885,7 +1960,7 @@ are always available. They are listed here in alphabetical order. the same data with other ordering tools such as :func:`max` that rely on a different underlying method. Implementing all six comparisons also helps avoid confusion for mixed type comparisons which can call - reflected the :meth:`~object.__gt__` method. + the reflected :meth:`~object.__gt__` method. For sorting examples and a brief sorting tutorial, see :ref:`sortinghowto`. @@ -1909,7 +1984,7 @@ are always available. They are listed here in alphabetical order. be used in the class definition (such as ``f()``). Static methods in Python are similar to those found in Java or C++. Also, see - :func:`classmethod` for a variant that is useful for creating alternate class + :deco:`classmethod` for a variant that is useful for creating alternate class constructors. Like all decorators, it is also possible to call ``staticmethod`` as @@ -1938,8 +2013,10 @@ are always available. They are listed here in alphabetical order. single: string; str() (built-in function) .. _func-str: -.. class:: str(object='') - str(object=b'', encoding='utf-8', errors='strict') +.. class:: str(*, encoding='utf-8', errors='strict') + str(object) + str(object, encoding, errors='strict') + str(object, *, errors) :noindex: Return a :class:`str` version of *object*. See :func:`str` for details. @@ -1972,7 +2049,7 @@ are always available. They are listed here in alphabetical order. .. class:: super() - super(type, object_or_type=None) + super(type, object_or_type=None, /) Return a proxy object that delegates method calls to a parent or sibling class of *type*. This is useful for accessing inherited methods that have @@ -2054,16 +2131,15 @@ are always available. They are listed here in alphabetical order. .. _func-tuple: -.. class:: tuple() - tuple(iterable) +.. class:: tuple(iterable=(), /) :noindex: Rather than being a function, :class:`tuple` is actually an immutable sequence type, as documented in :ref:`typesseq-tuple` and :ref:`typesseq`. -.. class:: type(object) - type(name, bases, dict, **kwds) +.. class:: type(object, /) + type(name, bases, dict, /, **kwargs) .. index:: pair: object; type @@ -2105,8 +2181,12 @@ are always available. They are listed here in alphabetical order. Subclasses of :class:`!type` which don't override ``type.__new__`` may no longer use the one-argument form to get the type of an object. + .. versionchanged:: 3.15 + + *dict* can now be a :class:`frozendict`. + .. function:: vars() - vars(object) + vars(object, /) Return the :attr:`~object.__dict__` attribute for a module, class, instance, or any other object with a :attr:`!__dict__` attribute. diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index beec9b942afc0f1..2b46978f058102b 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -4,13 +4,6 @@ .. module:: functools :synopsis: Higher-order functions and operations on callable objects. -.. moduleauthor:: Peter Harris -.. moduleauthor:: Raymond Hettinger -.. moduleauthor:: Nick Coghlan -.. moduleauthor:: Łukasz Langa -.. moduleauthor:: Pablo Galindo -.. sectionauthor:: Peter Harris - **Source code:** :source:`Lib/functools.py` .. testsetup:: default @@ -20,11 +13,11 @@ -------------- -The :mod:`functools` module is for higher-order functions: functions that act on +The :mod:`!functools` module is for higher-order functions: functions that act on or return other functions. In general, any callable object can be treated as a function for the purposes of this module. -The :mod:`functools` module defines the following functions: +The :mod:`!functools` module defines the following functions: .. decorator:: cache(user_function) @@ -34,7 +27,7 @@ The :mod:`functools` module defines the following functions: Returns the same as ``lru_cache(maxsize=None)``, creating a thin wrapper around a dictionary lookup for the function arguments. Because it never needs to evict old values, this is smaller and faster than - :func:`lru_cache` with a size limit. + :deco:`lru_cache` with a size limit. For example:: @@ -42,11 +35,11 @@ The :mod:`functools` module defines the following functions: def factorial(n): return n * factorial(n-1) if n else 1 - >>> factorial(10) # no previously cached result, makes 11 recursive calls + >>> factorial(10) # no previously cached result, makes 11 recursive calls 3628800 - >>> factorial(5) # just looks up cached value result + >>> factorial(5) # no new calls, just returns the cached result 120 - >>> factorial(12) # makes two new recursive calls, the other 10 are cached + >>> factorial(12) # two new recursive calls, factorial(10) is cached 479001600 The cache is threadsafe so that the wrapped function can be used in @@ -57,6 +50,10 @@ The :mod:`functools` module defines the following functions: another thread makes an additional call before the initial call has been completed and cached. + Call-once behavior is not guaranteed because locks are not held during the + function call. Potentially another call with the same arguments could + occur while the first call is still running. + .. versionadded:: 3.9 @@ -64,7 +61,7 @@ The :mod:`functools` module defines the following functions: Transform a method of a class into a property whose value is computed once and then cached as a normal attribute for the life of the instance. Similar - to :func:`property`, with the addition of caching. Useful for expensive + to :deco:`property`, with the addition of caching. Useful for expensive computed properties of instances that are otherwise effectively immutable. Example:: @@ -78,8 +75,8 @@ The :mod:`functools` module defines the following functions: def stdev(self): return statistics.stdev(self._data) - The mechanics of :func:`cached_property` are somewhat different from - :func:`property`. A regular property blocks attribute writes unless a + The mechanics of :deco:`cached_property` are somewhat different from + :deco:`property`. A regular property blocks attribute writes unless a setter is defined. In contrast, a *cached_property* allows writes. The *cached_property* decorator only runs on lookups and only when an @@ -111,14 +108,14 @@ The :mod:`functools` module defines the following functions: (as such classes don't provide a ``__dict__`` attribute at all). If a mutable mapping is not available or if space-efficient key sharing is - desired, an effect similar to :func:`cached_property` can also be achieved by - stacking :func:`property` on top of :func:`lru_cache`. See - :ref:`faq-cache-method-calls` for more details on how this differs from :func:`cached_property`. + desired, an effect similar to :deco:`cached_property` can also be achieved by + stacking :deco:`property` on top of :deco:`lru_cache`. See + :ref:`faq-cache-method-calls` for more details on how this differs from :deco:`cached_property`. .. versionadded:: 3.8 .. versionchanged:: 3.12 - Prior to Python 3.12, ``cached_property`` included an undocumented lock to + Prior to Python 3.12, :deco:`!cached_property` included an undocumented lock to ensure that in multi-threaded usage the getter function was guaranteed to run only once per instance. However, the lock was per-property, not per-instance, which could result in unacceptably high lock contention. In @@ -176,8 +173,8 @@ The :mod:`functools` module defines the following functions: the *maxsize* at its default value of 128:: @lru_cache - def count_vowels(sentence): - return sum(sentence.count(vowel) for vowel in 'AEIOUaeiou') + def count_vowels(word): + return sum(word.count(vowel) for vowel in 'AEIOUaeiou') If *maxsize* is set to ``None``, the LRU feature is disabled and the cache can grow without bound. @@ -190,7 +187,7 @@ The :mod:`functools` module defines the following functions: Note, type specificity applies only to the function's immediate arguments rather than their contents. The scalar arguments, ``Decimal(42)`` and - ``Fraction(42)`` are be treated as distinct calls with distinct results. + ``Fraction(42)`` are treated as distinct calls with distinct results. In contrast, the tuple arguments ``('answer', Decimal(42))`` and ``('answer', Fraction(42))`` are treated as equivalent. @@ -471,7 +468,7 @@ The :mod:`functools` module defines the following functions: Roughly equivalent to:: - initial_missing = object() + initial_missing = sentinel('initial_missing') def reduce(function, iterable, /, initial=initial_missing): it = iter(iterable) @@ -672,7 +669,7 @@ The :mod:`functools` module defines the following functions: dispatch>` :term:`generic function`. To define a generic method, decorate it with the ``@singledispatchmethod`` - decorator. When defining a function using ``@singledispatchmethod``, note + decorator. When defining a method using ``@singledispatchmethod``, note that the dispatch happens on the type of the first non-*self* or non-*cls* argument:: @@ -690,7 +687,7 @@ The :mod:`functools` module defines the following functions: return not arg ``@singledispatchmethod`` supports nesting with other decorators such as - :func:`@classmethod`. Note that to allow for + :deco:`classmethod`. Note that to allow for ``dispatcher.register``, ``singledispatchmethod`` must be the *outer most* decorator. Here is the ``Negator`` class with the ``neg`` methods bound to the class, rather than an instance of the class:: @@ -712,11 +709,13 @@ The :mod:`functools` module defines the following functions: return not arg The same pattern can be used for other similar decorators: - :func:`@staticmethod`, - :func:`@abstractmethod`, and others. + :deco:`staticmethod`, :deco:`~abc.abstractmethod`, and others. .. versionadded:: 3.8 + .. versionchanged:: 3.15 + Added support of non-:term:`descriptor` callables. + .. function:: update_wrapper(wrapper, wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) @@ -733,7 +732,7 @@ The :mod:`functools` module defines the following functions: function's :attr:`~function.__dict__`, i.e. the instance dictionary). To allow access to the original function for introspection and other purposes - (e.g. bypassing a caching decorator such as :func:`lru_cache`), this function + (e.g. bypassing a caching decorator such as :deco:`lru_cache`), this function automatically adds a ``__wrapped__`` attribute to the wrapper that refers to the function being wrapped. diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 2ef5c4b35a25cc0..65533e7e57adc33 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -4,9 +4,6 @@ .. module:: gc :synopsis: Interface to the cycle-detecting garbage collector. -.. moduleauthor:: Neil Schemenauer -.. sectionauthor:: Neil Schemenauer - -------------- This module provides an interface to the optional garbage collector. It @@ -20,7 +17,7 @@ can be disabled by calling ``gc.disable()``. To debug a leaking program call ``gc.DEBUG_SAVEALL``, causing garbage-collected objects to be saved in gc.garbage for inspection. -The :mod:`gc` module provides the following functions: +The :mod:`!gc` module provides the following functions: .. function:: enable() @@ -40,18 +37,11 @@ The :mod:`gc` module provides the following functions: .. function:: collect(generation=2) - Perform a collection. The optional argument *generation* + With no arguments, run a full collection. The optional argument *generation* may be an integer specifying which generation to collect (from 0 to 2). A :exc:`ValueError` is raised if the generation number is invalid. The sum of collected objects and uncollectable objects is returned. - Calling ``gc.collect(0)`` will perform a GC collection on the young generation. - - Calling ``gc.collect(1)`` will perform a GC collection on the young generation - and an increment of the old generation. - - Calling ``gc.collect(2)`` or ``gc.collect()`` performs a full collection - The free lists maintained for a number of built-in types are cleared whenever a full collection or collection of the highest generation (2) is run. Not all items in some free lists may be freed due to the @@ -63,6 +53,9 @@ The :mod:`gc` module provides the following functions: .. versionchanged:: 3.14 ``generation=1`` performs an increment of collection. + .. versionchanged:: 3.14.5 + ``generation=1`` performs collection of the middle generation. + .. function:: set_debug(flags) @@ -78,13 +71,9 @@ The :mod:`gc` module provides the following functions: .. function:: get_objects(generation=None) - Returns a list of all objects tracked by the collector, excluding the list - returned. If *generation* is not ``None``, return only the objects as follows: - - * 0: All objects in the young generation - * 1: No objects, as there is no generation 1 (as of Python 3.14) - * 2: All objects in the old generation + returned. If *generation* is not ``None``, return only the objects tracked by + the collector that are in that generation. .. versionchanged:: 3.8 New *generation* parameter. @@ -92,6 +81,9 @@ The :mod:`gc` module provides the following functions: .. versionchanged:: 3.14 Generation 1 is removed + .. versionchanged:: 3.14.5 + Generation 1 is reintroduced to maintain GC behavior from 3.13. + .. audit-event:: gc.get_objects generation gc.get_objects .. function:: get_stats() @@ -108,43 +100,47 @@ The :mod:`gc` module provides the following functions: * ``uncollectable`` is the total number of objects which were found to be uncollectable (and were therefore moved to the :data:`garbage` - list) inside this generation. + list) inside this generation; + + * ``candidates`` is the total number of objects in this generation which were + considered for collection and traversed; + + * ``duration`` is the total time in seconds spent in collections for this + generation. .. versionadded:: 3.4 + .. versionchanged:: 3.15 + Add ``duration`` and ``candidates``. + .. function:: set_threshold(threshold0, [threshold1, [threshold2]]) Set the garbage collection thresholds (the collection frequency). Setting *threshold0* to zero disables collection. - The GC classifies objects into two generations depending on whether they have - survived a collection. New objects are placed in the young generation. If an - object survives a collection it is moved into the old generation. - - In order to decide when to run, the collector keeps track of the number of object + The GC classifies objects into three generations depending on how many + collection sweeps they have survived. New objects are placed in the youngest + generation (generation ``0``). If an object survives a collection it is moved + into the next older generation. Since generation ``2`` is the oldest + generation, objects in that generation remain there after a collection. In + order to decide when to run, the collector keeps track of the number object allocations and deallocations since the last collection. When the number of allocations minus the number of deallocations exceeds *threshold0*, collection - starts. For each collection, all the objects in the young generation and some - fraction of the old generation is collected. - - In the free-threaded build, the increase in process memory usage is also - checked before running the collector. If the memory usage has not increased - by 10% since the last collection and the net number of object allocations - has not exceeded 40 times *threshold0*, the collection is not run. - - The fraction of the old generation that is collected is **inversely** proportional - to *threshold1*. The larger *threshold1* is, the slower objects in the old generation - are collected. - For the default value of 10, 1% of the old generation is scanned during each collection. - - *threshold2* is ignored. + starts. Initially only generation ``0`` is examined. If generation ``0`` has + been examined more than *threshold1* times since generation ``1`` has been + examined, then generation ``1`` is examined as well. + With the third generation, things are a bit more complicated, + see `Collecting the oldest generation `_ for more information. - See `Garbage collector design `_ for more information. + See `Garbage collector design `_ for more information. .. versionchanged:: 3.14 *threshold2* is ignored + .. versionchanged:: 3.14.5 + *threshold2* is restored to match Python 3.13 behavior. + .. function:: get_count() @@ -313,6 +309,12 @@ values but should not rebind them): "uncollectable": When *phase* is "stop", the number of objects that could not be collected and were put in :data:`garbage`. + "candidates": When *phase* is "stop", the total number of objects in this + generation which were considered for collection and traversed. + + "duration": When *phase* is "stop", the time in seconds spent in the + collection. + Applications can add their own callbacks to this list. The primary use cases are: @@ -325,6 +327,9 @@ values but should not rebind them): .. versionadded:: 3.3 + .. versionchanged:: 3.15 + Add "duration" and "candidates". + The following constants are provided for use with :func:`set_debug`: diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index 0fb0fc88683c032..f60cc63414f764d 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -4,17 +4,13 @@ .. module:: getpass :synopsis: Portable reading of passwords and retrieval of the userid. -.. moduleauthor:: Piers Lauder -.. sectionauthor:: Fred L. Drake, Jr. -.. Windows (& Mac?) support by Guido van Rossum. - **Source code:** :source:`Lib/getpass.py` -------------- .. include:: ../includes/wasm-notavail.rst -The :mod:`getpass` module provides two functions: +The :mod:`!getpass` module provides two functions: .. function:: getpass(prompt='Password: ', stream=None, *, echo_char=None) @@ -22,26 +18,48 @@ The :mod:`getpass` module provides two functions: the string *prompt*, which defaults to ``'Password: '``. On Unix, the prompt is written to the file-like object *stream* using the replace error handler if needed. *stream* defaults to the controlling terminal - (:file:`/dev/tty`) or if that is unavailable to ``sys.stderr`` (this + (:file:`/dev/tty`) or if that is unavailable to :data:`sys.stderr` (this argument is ignored on Windows). The *echo_char* argument controls how user input is displayed while typing. If *echo_char* is ``None`` (default), input remains hidden. Otherwise, - *echo_char* must be a printable ASCII string and each typed character - is replaced by it. For example, ``echo_char='*'`` will display - asterisks instead of the actual input. + *echo_char* must be a single printable ASCII character and each + typed character is replaced by it. For example, ``echo_char='*'`` will + display asterisks instead of the actual input. - If echo free input is unavailable getpass() falls back to printing - a warning message to *stream* and reading from ``sys.stdin`` and + If echo-free input is unavailable, :func:`getpass` falls back to printing + a warning message to *stream* and reading from :data:`sys.stdin` and issuing a :exc:`GetPassWarning`. .. note:: - If you call getpass from within IDLE, the input may be done in the + If you call :func:`getpass` from within IDLE, the input may be done in the terminal you launched IDLE from rather than the idle window itself. + .. note:: + On Unix systems, when *echo_char* is set, the terminal will be + configured to operate in + :manpage:`noncanonical mode `. + Common terminal control characters are supported: + + * :kbd:`Ctrl+A` - Move cursor to beginning of line + * :kbd:`Ctrl+E` - Move cursor to end of line + * :kbd:`Ctrl+K` - Kill (delete) from cursor to end of line + * :kbd:`Ctrl+U` - Kill (delete) entire line + * :kbd:`Ctrl+W` - Erase previous word + * :kbd:`Ctrl+V` - Insert next character literally (quote) + * :kbd:`Backspace`/:kbd:`DEL` - Delete character before cursor + + These shortcuts work by reading the terminal's configured control + character mappings from termios settings. + .. versionchanged:: 3.14 Added the *echo_char* parameter for keyboard feedback. + .. versionchanged:: 3.15 + When using non-empty *echo_char* on Unix, keyboard shortcuts (including + cursor movement and line editing) are now properly handled using the + terminal's control character configuration. + .. exception:: GetPassWarning A :exc:`UserWarning` subclass issued when password input may be echoed. diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index d0de83907eb2979..2ab7ba7df19cf14 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -4,14 +4,11 @@ .. module:: gettext :synopsis: Multilingual internationalization services. -.. moduleauthor:: Barry A. Warsaw -.. sectionauthor:: Barry A. Warsaw - **Source code:** :source:`Lib/gettext.py` -------------- -The :mod:`gettext` module provides internationalization (I18N) and localization +The :mod:`!gettext` module provides internationalization (I18N) and localization (L10N) services for your Python modules and applications. It supports both the GNU :program:`gettext` message catalog API and a higher level, class-based API that may be more appropriate for Python files. The interface described below allows you @@ -25,7 +22,7 @@ Some hints on localizing your Python modules and applications are also given. GNU :program:`gettext` API -------------------------- -The :mod:`gettext` module defines the following API, which is very similar to +The :mod:`!gettext` module defines the following API, which is very similar to the GNU :program:`gettext` API. If you use this API you will affect the translation of your entire application globally. Often this is what you want if your application is monolingual, with the choice of language dependent on the @@ -37,7 +34,7 @@ class-based API instead. .. function:: bindtextdomain(domain, localedir=None) Bind the *domain* to the locale directory *localedir*. More concretely, - :mod:`gettext` will look for binary :file:`.mo` files for the given domain using + :mod:`!gettext` will look for binary :file:`.mo` files for the given domain using the path (on Unix): :file:`{localedir}/{language}/LC_MESSAGES/{domain}.mo`, where *language* is searched for in the environment variables :envvar:`LANGUAGE`, :envvar:`LC_ALL`, :envvar:`LC_MESSAGES`, and :envvar:`LANG` respectively. @@ -54,19 +51,19 @@ class-based API instead. .. index:: single: _ (underscore); gettext -.. function:: gettext(message) +.. function:: gettext(message, /) Return the localized translation of *message*, based on the current global domain, language, and locale directory. This function is usually aliased as :func:`!_` in the local namespace (see examples below). -.. function:: dgettext(domain, message) +.. function:: dgettext(domain, message, /) Like :func:`.gettext`, but look the message up in the specified *domain*. -.. function:: ngettext(singular, plural, n) +.. function:: ngettext(singular, plural, n, /) Like :func:`.gettext`, but consider plural forms. If a translation is found, apply the plural formula to *n*, and return the resulting message (some @@ -81,15 +78,15 @@ class-based API instead. formulas for a variety of languages. -.. function:: dngettext(domain, singular, plural, n) +.. function:: dngettext(domain, singular, plural, n, /) Like :func:`ngettext`, but look the message up in the specified *domain*. -.. function:: pgettext(context, message) -.. function:: dpgettext(domain, context, message) -.. function:: npgettext(context, singular, plural, n) -.. function:: dnpgettext(domain, context, singular, plural, n) +.. function:: pgettext(context, message, /) +.. function:: dpgettext(domain, context, message, /) +.. function:: npgettext(context, singular, plural, n, /) +.. function:: dnpgettext(domain, context, singular, plural, n, /) Similar to the corresponding functions without the ``p`` in the prefix (that is, :func:`gettext`, :func:`dgettext`, :func:`ngettext`, :func:`dngettext`), @@ -114,7 +111,7 @@ Here's an example of typical usage for this API:: Class-based API --------------- -The class-based API of the :mod:`gettext` module gives you more flexibility and +The class-based API of the :mod:`!gettext` module gives you more flexibility and greater convenience than the GNU :program:`gettext` API. It is the recommended way of localizing your Python applications and modules. :mod:`!gettext` defines a :class:`GNUTranslations` class which implements the parsing of GNU :file:`.mo` format @@ -226,20 +223,20 @@ are the methods of :class:`!NullTranslations`: translation for a given message. - .. method:: gettext(message) + .. method:: gettext(message, /) If a fallback has been set, forward :meth:`!gettext` to the fallback. Otherwise, return *message*. Overridden in derived classes. - .. method:: ngettext(singular, plural, n) + .. method:: ngettext(singular, plural, n, /) If a fallback has been set, forward :meth:`!ngettext` to the fallback. Otherwise, return *singular* if *n* is 1; return *plural* otherwise. Overridden in derived classes. - .. method:: pgettext(context, message) + .. method:: pgettext(context, message, /) If a fallback has been set, forward :meth:`pgettext` to the fallback. Otherwise, return the translated message. Overridden in derived classes. @@ -247,7 +244,7 @@ are the methods of :class:`!NullTranslations`: .. versionadded:: 3.8 - .. method:: npgettext(context, singular, plural, n) + .. method:: npgettext(context, singular, plural, n, /) If a fallback has been set, forward :meth:`npgettext` to the fallback. Otherwise, return the translated message. Overridden in derived classes. @@ -325,7 +322,7 @@ unexpected, or if other problems occur while reading the file, instantiating a The following methods are overridden from the base class implementation: - .. method:: gettext(message) + .. method:: gettext(message, /) Look up the *message* id in the catalog and return the corresponding message string, as a Unicode string. If there is no entry in the catalog for the @@ -334,7 +331,7 @@ unexpected, or if other problems occur while reading the file, instantiating a *message* id is returned. - .. method:: ngettext(singular, plural, n) + .. method:: ngettext(singular, plural, n, /) Do a plural-forms lookup of a message id. *singular* is used as the message id for purposes of lookup in the catalog, while *n* is used to determine which @@ -355,7 +352,7 @@ unexpected, or if other problems occur while reading the file, instantiating a n) % {'num': n} - .. method:: pgettext(context, message) + .. method:: pgettext(context, message, /) Look up the *context* and *message* id in the catalog and return the corresponding message string, as a Unicode string. If there is no @@ -366,7 +363,7 @@ unexpected, or if other problems occur while reading the file, instantiating a .. versionadded:: 3.8 - .. method:: npgettext(context, singular, plural, n) + .. method:: npgettext(context, singular, plural, n, /) Do a plural-forms lookup of a message id. *singular* is used as the message id for purposes of lookup in the catalog, while *n* is used to @@ -393,7 +390,7 @@ The Catalog constructor .. index:: single: GNOME -GNOME uses a version of the :mod:`gettext` module by James Henstridge, but this +GNOME uses a version of the :mod:`!gettext` module by James Henstridge, but this version has a slightly different API. Its documented usage was:: import gettext @@ -425,7 +422,7 @@ take the following steps: #. create language-specific translations of the message catalogs -#. use the :mod:`gettext` module so that message strings are properly translated +#. use the :mod:`!gettext` module so that message strings are properly translated In order to prepare your code for I18N, you need to look at all the strings in your files. Any string that needs to be translated should be marked by wrapping @@ -473,10 +470,10 @@ supported natural language. They send back the completed language-specific versions as a :file:`.po` file that's compiled into a machine-readable :file:`.mo` binary catalog file using the :program:`msgfmt` program. The :file:`.mo` files are used by the -:mod:`gettext` module for the actual translation processing at +:mod:`!gettext` module for the actual translation processing at run-time. -How you use the :mod:`gettext` module in your code depends on whether you are +How you use the :mod:`!gettext` module in your code depends on whether you are internationalizing a single module or your entire application. The next two sections will discuss each case. diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 59ad1b07f273380..942f23d216fc075 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -18,23 +18,27 @@ single: - (minus); in glob-style wildcards single: . (dot); in glob-style wildcards -The :mod:`glob` module finds all the pathnames matching a specified pattern -according to the rules used by the Unix shell, although results are returned in -arbitrary order. No tilde expansion is done, but ``*``, ``?``, and character +The :mod:`!glob` module finds pathnames +using pattern matching rules similar to the Unix shell. +No tilde expansion is done, but ``*``, ``?``, and character ranges expressed with ``[]`` will be correctly matched. This is done by using the :func:`os.scandir` and :func:`fnmatch.fnmatch` functions in concert, and not by actually invoking a subshell. -Note that files beginning with a dot (``.``) can only be matched by +.. note:: + The pathnames are returned in no particular order. If you need a specific + order, sort the results. + +Files beginning with a dot (``.``) can only be matched by patterns that also start with a dot, unlike :func:`fnmatch.fnmatch` or :func:`pathlib.Path.glob`. -(For tilde and shell variable expansion, use :func:`os.path.expanduser` and -:func:`os.path.expandvars`.) +For tilde and shell variable expansion, use :func:`os.path.expanduser` and +:func:`os.path.expandvars`. For a literal match, wrap the meta-characters in brackets. For example, ``'[?]'`` matches the character ``'?'``. -The :mod:`glob` module defines the following functions: +The :mod:`!glob` module defines the following functions: .. function:: glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, \ @@ -51,7 +55,7 @@ The :mod:`glob` module defines the following functions: If *root_dir* is not ``None``, it should be a :term:`path-like object` specifying the root directory for searching. It has the same effect on - :func:`glob` as changing the current directory before calling it. If + :func:`!glob` as changing the current directory before calling it. If *pathname* is relative, the result will contain paths relative to *root_dir*. @@ -79,6 +83,11 @@ The :mod:`glob` module defines the following functions: This function may return duplicate path names if *pathname* contains multiple "``**``" patterns and *recursive* is true. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -102,6 +111,11 @@ The :mod:`glob` module defines the following functions: This function may return duplicate path names if *pathname* contains multiple "``**``" patterns and *recursive* is true. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -126,7 +140,8 @@ The :mod:`glob` module defines the following functions: .. function:: translate(pathname, *, recursive=False, include_hidden=False, seps=None) Convert the given path specification to a regular expression for use with - :func:`re.match`. The path specification can contain shell-style wildcards. + :func:`re.prefixmatch`. The path specification can contain shell-style + wildcards. For example: @@ -136,7 +151,7 @@ The :mod:`glob` module defines the following functions: >>> regex '(?s:(?:.+/)?[^/]*\\.txt)\\z' >>> reobj = re.compile(regex) - >>> reobj.match('foo/bar/baz.txt') + >>> reobj.prefixmatch('foo/bar/baz.txt') Path separators and segments are meaningful to this function, unlike diff --git a/Doc/library/graphlib.rst b/Doc/library/graphlib.rst index 053d5f8231ba0e3..21f4d1fb938038e 100644 --- a/Doc/library/graphlib.rst +++ b/Doc/library/graphlib.rst @@ -204,7 +204,7 @@ Exceptions ---------- -The :mod:`graphlib` module defines the following exception classes: +The :mod:`!graphlib` module defines the following exception classes: .. exception:: CycleError diff --git a/Doc/library/grp.rst b/Doc/library/grp.rst index d1c7f22a2097803..f436970e791ace1 100644 --- a/Doc/library/grp.rst +++ b/Doc/library/grp.rst @@ -2,7 +2,6 @@ ================================== .. module:: grp - :platform: Unix :synopsis: The group database (getgrnam() and friends). -------------- diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index c59014a6f5bdce5..9211e5f18c6b6e6 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -11,9 +11,11 @@ This module provides a simple interface to compress and decompress files just like the GNU programs :program:`gzip` and :program:`gunzip` would. +.. include:: ../includes/optional-module.rst + The data compression is provided by the :mod:`zlib` module. -The :mod:`gzip` module provides the :class:`GzipFile` class, as well as the +The :mod:`!gzip` module provides the :class:`GzipFile` class, as well as the :func:`.open`, :func:`compress` and :func:`decompress` convenience functions. The :class:`GzipFile` class reads and writes :program:`gzip`\ -format files, automatically compressing or decompressing the data so that it looks like an @@ -26,7 +28,7 @@ Note that additional file formats which can be decompressed by the The module defines the following items: -.. function:: open(filename, mode='rb', compresslevel=6, encoding=None, errors=None, newline=None) +.. function:: open(filename, mode='rb', compresslevel=6, encoding=None, errors=None, newline=None, *, mtime=None) Open a gzip-compressed file in binary or text mode, returning a :term:`file object`. @@ -41,9 +43,12 @@ The module defines the following items: The *compresslevel* argument is an integer from 0 to 9, as for the :class:`GzipFile` constructor. + The keyword-only argument *mtime* represents a Unix timestamp. + For binary mode, this function is equivalent to the :class:`GzipFile` - constructor: ``GzipFile(filename, mode, compresslevel)``. In this case, the - *encoding*, *errors* and *newline* arguments must not be provided. + constructor: ``GzipFile(filename, mode, compresslevel, mtime=mtime)``. + In this case, the *encoding*, *errors* and *newline* arguments must not + be provided. For text mode, a :class:`GzipFile` object is created, and wrapped in an :class:`io.TextIOWrapper` instance with the specified encoding, error @@ -59,11 +64,15 @@ The module defines the following items: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. versionchanged:: next + .. versionchanged:: 3.15 The default compression level was reduced to 6 (down from 9). It is the default level used by most compression tools and a better tradeoff between speed and performance. + .. versionchanged:: next + Added keyword-only argument *mtime* which is passed to the class + constructor of :class:`~gzip.GzipFile`. + .. exception:: BadGzipFile An exception raised for invalid gzip files. It inherits from :exc:`OSError`. @@ -106,9 +115,13 @@ The module defines the following items: is no compression. The default is ``9``. The optional *mtime* argument is the timestamp requested by gzip. The time - is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. - If *mtime* is omitted or ``None``, the current time is used. Use *mtime* = 0 - to generate a compressed stream that does not depend on creation time. + is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. Set + *mtime* to ``0`` to generate a compressed stream that does not depend on + creation time. If *mtime* is omitted or ``None``, the current time is used; + however, if the current time is outside the range 00:00:00 UTC, January 1, + 1970 through 06:28:15 UTC, February 7, 2106, or explicitly passed *mtime* + argument is outside the range ``0`` to ``2**32-1``, then the value ``0`` + is used instead. See below for the :attr:`mtime` attribute that is set when decompressing. @@ -186,7 +199,7 @@ The module defines the following items: Remove the ``filename`` attribute, use the :attr:`~GzipFile.name` attribute instead. - .. versionchanged:: next + .. versionchanged:: 3.15 The default compression level was reduced to 6 (down from 9). It is the default level used by most compression tools and a better tradeoff between speed and performance. @@ -216,7 +229,7 @@ The module defines the following items: The *mtime* parameter now defaults to 0 for reproducible output. For the previous behaviour of using the current time, pass ``None`` to *mtime*. - .. versionchanged:: next + .. versionchanged:: 3.15 The default compression level was reduced to 6 (down from 9). It is the default level used by most compression tools and a better tradeoff between speed and performance. @@ -281,20 +294,20 @@ Example of how to GZIP compress a binary string:: .. _gzip-cli: -Command Line Interface +Command-line interface ---------------------- -The :mod:`gzip` module provides a simple command line interface to compress or +The :mod:`!gzip` module provides a simple command line interface to compress or decompress files. -Once executed the :mod:`gzip` module keeps the input file(s). +Once executed the :mod:`!gzip` module keeps the input file(s). .. versionchanged:: 3.8 Add a new command line interface with a usage. By default, when you will execute the CLI, the default compression level is 6. -Command line options +Command-line options ^^^^^^^^^^^^^^^^^^^^ .. option:: file diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 8bba6700930bf4e..ed0b0b2735b5c3d 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -4,9 +4,6 @@ .. module:: hashlib :synopsis: Secure hash and message digest algorithms. -.. moduleauthor:: Gregory P. Smith -.. sectionauthor:: Gregory P. Smith - **Source code:** :source:`Lib/hashlib.py` .. index:: @@ -61,7 +58,7 @@ if you are using a rare "FIPS compliant" build of Python. These correspond to :data:`algorithms_guaranteed`. Additional algorithms may also be available if your Python distribution's -:mod:`hashlib` was linked against a build of OpenSSL that provides others. +:mod:`!hashlib` was linked against a build of OpenSSL that provides others. Others *are not guaranteed available* on all installations and will only be accessible by name via :func:`new`. See :data:`algorithms_available`. @@ -310,7 +307,7 @@ a file or file-like object. .. versionadded:: 3.11 .. versionchanged:: 3.14 - Now raises a :exc:`BlockingIOError` if the file is opened in blocking + Now raises a :exc:`BlockingIOError` if the file is opened in non-blocking mode. Previously, spurious null bytes were added to the digest. @@ -379,8 +376,6 @@ include a `salt `_. BLAKE2 ------ -.. sectionauthor:: Dmitry Chestnykh - .. index:: single: blake2b, blake2s @@ -397,7 +392,7 @@ BLAKE2 supports **keyed mode** (a faster and simpler replacement for HMAC_), **salted hashing**, **personalization**, and **tree hashing**. Hash objects from this module follow the API of standard library's -:mod:`hashlib` objects. +:mod:`!hashlib` objects. Creating hash objects diff --git a/Doc/library/heapq.rst b/Doc/library/heapq.rst index 462b65bc7afba48..26cffa7c643028a 100644 --- a/Doc/library/heapq.rst +++ b/Doc/library/heapq.rst @@ -4,11 +4,6 @@ .. module:: heapq :synopsis: Heap queue algorithm (a.k.a. priority queue). -.. moduleauthor:: Kevin O'Connor -.. sectionauthor:: Guido van Rossum -.. sectionauthor:: François Pinard -.. sectionauthor:: Raymond Hettinger - **Source code:** :source:`Lib/heapq.py` -------------- @@ -58,6 +53,11 @@ functions, respectively. The following functions are provided for min-heaps: +.. function:: heapify(x) + + Transform list *x* into a min-heap, in-place, in linear time. + + .. function:: heappush(heap, item) Push the value *item* onto the *heap*, maintaining the min-heap invariant. @@ -77,11 +77,6 @@ The following functions are provided for min-heaps: followed by a separate call to :func:`heappop`. -.. function:: heapify(x) - - Transform list *x* into a min-heap, in-place, in linear time. - - .. function:: heapreplace(heap, item) Pop and return the smallest item from the *heap*, and also push the new *item*. @@ -231,6 +226,42 @@ Heap elements can be tuples. This is useful for assigning comparison values (1, 'write spec') +Other Applications +------------------ + +`Medians `_ are a measure of +central tendency for a set of numbers. In distributions skewed by +outliers, the median provides a more stable estimate than an average +(arithmetic mean). A running median is an `online algorithm +`_ that updates +continuously as new data arrives. + +A running median can be efficiently implemented by balancing two heaps, +a max-heap for values at or below the midpoint and a min-heap for values +above the midpoint. When the two heaps have the same size, the new +median is the average of the tops of the two heaps; otherwise, the +median is at the top of the larger heap:: + + def running_median(iterable): + "Yields the cumulative median of values seen so far." + + lo = [] # max-heap + hi = [] # min-heap (same size as or one smaller than lo) + + for x in iterable: + if len(lo) == len(hi): + heappush_max(lo, heappushpop(hi, x)) + yield lo[0] + else: + heappush(hi, heappushpop_max(lo, x)) + yield (lo[0] + hi[0]) / 2 + +For example:: + + >>> list(running_median([5.0, 9.0, 4.0, 12.0, 8.0, 9.0])) + [5.0, 7.0, 5.0, 7.0, 8.0, 8.5] + + Priority Queue Implementation Notes ----------------------------------- diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index 57076c38086c79b..2ee0c0bd9128b89 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -4,9 +4,6 @@ .. module:: hmac :synopsis: Keyed-Hashing for Message Authentication (HMAC) implementation -.. moduleauthor:: Gerhard Häring -.. sectionauthor:: Gerhard Häring - **Source code:** :source:`Lib/hmac.py` -------------- @@ -50,7 +47,9 @@ cannot be used with HMAC. .. versionadded:: 3.7 -An HMAC object has the following methods: +.. class:: HMAC + + An HMAC object has the following methods: .. method:: HMAC.update(msg) diff --git a/Doc/library/html.entities.rst b/Doc/library/html.entities.rst index add18e4c87d2204..15d2dc2e9aa6bc9 100644 --- a/Doc/library/html.entities.rst +++ b/Doc/library/html.entities.rst @@ -4,8 +4,6 @@ .. module:: html.entities :synopsis: Definitions of HTML general entities. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/html/entities.py` -------------- diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst index dd67fc34e856f1d..11f851d4f6c4b74 100644 --- a/Doc/library/html.parser.rst +++ b/Doc/library/html.parser.rst @@ -15,14 +15,18 @@ This module defines a class :class:`HTMLParser` which serves as the basis for parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML. -.. class:: HTMLParser(*, convert_charrefs=True) +.. class:: HTMLParser(*, convert_charrefs=True, scripting=False) Create a parser instance able to parse invalid markup. - If *convert_charrefs* is ``True`` (the default), all character - references (except the ones in ``script``/``style`` elements) are + If *convert_charrefs* is true (the default), all character + references (except the ones in elements like ``script`` and ``style``) are automatically converted to the corresponding Unicode characters. + If *scripting* is false (the default), the content of the ``noscript`` + element is parsed normally; if it's true, it's returned as is without + being parsed. + An :class:`.HTMLParser` instance is fed HTML data and calls handler methods when start tags, end tags, text, comments, and other markup elements are encountered. The user should subclass :class:`.HTMLParser` and override its @@ -37,6 +41,9 @@ parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML. .. versionchanged:: 3.5 The default value for argument *convert_charrefs* is now ``True``. + .. versionchanged:: 3.14.1 + Added the *scripting* parameter. + Example HTML Parser Application ------------------------------- @@ -134,7 +141,7 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): argument is a list of ``(name, value)`` pairs containing the attributes found inside the tag's ``<>`` brackets. The *name* will be translated to lower case, and quotes in the *value* have been removed, and character and entity references - have been replaced. + have been replaced. For empty attributes, *value* is ``None``. For instance, for the tag ````, this method would be called as ``handle_starttag('a', [('href', 'https://www.cwi.nl/')])``. @@ -161,15 +168,15 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): .. method:: HTMLParser.handle_data(data) This method is called to process arbitrary data (e.g. text nodes and the - content of ```` and ````). + content of elements like ``script`` and ``style``). .. method:: HTMLParser.handle_entityref(name) This method is called to process a named character reference of the form ``&name;`` (e.g. ``>``), where *name* is a general entity reference - (e.g. ``'gt'``). This method is never called if *convert_charrefs* is - ``True``. + (e.g. ``'gt'``). + This method is only called if *convert_charrefs* is false. .. method:: HTMLParser.handle_charref(name) @@ -177,8 +184,8 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): This method is called to process decimal and hexadecimal numeric character references of the form :samp:`&#{NNN};` and :samp:`&#x{NNN};`. For example, the decimal equivalent for ``>`` is ``>``, whereas the hexadecimal is ``>``; - in this case the method will receive ``'62'`` or ``'x3E'``. This method - is never called if *convert_charrefs* is ``True``. + in this case the method will receive ``'62'`` or ``'x3E'``. + This method is only called if *convert_charrefs* is false. .. method:: HTMLParser.handle_comment(data) @@ -292,8 +299,8 @@ Parsing an element with a few attributes and a title: Data : Python End tag : h1 -The content of ``script`` and ``style`` elements is returned as is, without -further parsing: +The content of elements like ``script`` and ``style`` is returned as is, +without further parsing: .. doctest:: @@ -304,12 +311,24 @@ further parsing: End tag : style >>> parser.feed('') + ... 'alert("hello! ☺");') Start tag: script attr: ('type', 'text/javascript') - Data : alert("hello!"); + Data : alert("hello! ☺"); End tag : script +Attribute names are converted to lowercase, quotes from attribute values removed, +and ``None`` is returned as *value* for empty attributes (such as ``checked``): + +.. doctest:: + + >>> parser.feed("") + Start tag: input + attr: ('type', 'checkbox') + attr: ('checked', None) + attr: ('required', '') + attr: ('disabled', 'disabled') + Parsing comments: .. doctest:: @@ -336,7 +355,7 @@ correct char (note: these 3 references are all equivalent to ``'>'``): Feeding incomplete chunks to :meth:`~HTMLParser.feed` works, but :meth:`~HTMLParser.handle_data` might be called more than once -(unless *convert_charrefs* is set to ``True``): +if *convert_charrefs* is false: .. doctest:: diff --git a/Doc/library/html.rst b/Doc/library/html.rst index 9aa39ba9a42b0fd..65c49a4107a0487 100644 --- a/Doc/library/html.rst +++ b/Doc/library/html.rst @@ -14,9 +14,12 @@ This module defines utilities to manipulate HTML. Convert the characters ``&``, ``<`` and ``>`` in string *s* to HTML-safe sequences. Use this if you need to display text that might contain such - characters in HTML. If the optional flag *quote* is true, the characters - (``"``) and (``'``) are also translated; this helps for inclusion in an HTML - attribute value delimited by quotes, as in ````. + characters in HTML. If the optional flag *quote* is true (the default), the + characters (``"``) and (``'``) are also translated; this helps for inclusion + in an HTML attribute value delimited by quotes, as in ````. + If *quote* is set to false, the characters (``"``) and (``'``) are not + translated. + .. versionadded:: 3.2 diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 07f5ebf57c9b543..ddf3d40d221fcd7 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -68,7 +68,7 @@ The module provides the following classes: .. versionchanged:: 3.7 *blocksize* parameter was added. - .. versionchanged:: next + .. versionchanged:: 3.15 *max_response_headers* parameter was added. @@ -114,7 +114,7 @@ The module provides the following classes: The deprecated *key_file*, *cert_file* and *check_hostname* parameters have been removed. - .. versionchanged:: next + .. versionchanged:: 3.15 *max_response_headers* parameter was added. @@ -133,7 +133,7 @@ This module provides the following function: Parse the headers from a file pointer *fp* representing a HTTP request/response. The file has to be a :class:`~io.BufferedIOBase` reader - (i.e. not text) and must provide a valid :rfc:`2822` style header. + (i.e. not text) and must provide a valid :rfc:`5322` style header. This function returns an instance of :class:`http.client.HTTPMessage` that holds the header fields, but no payload @@ -319,6 +319,12 @@ HTTPConnection Objects :class:`str` or bytes-like object that is not also a file as the body representation. + .. note:: + + Note that you must have read the whole response or call :meth:`close` + if :meth:`getresponse` raised an non-:exc:`ConnectionError` exception + before you can send a new request to the server. + .. versionchanged:: 3.2 *body* can now be an iterable. @@ -334,16 +340,15 @@ HTTPConnection Objects Should be called after a request is sent to get the response from the server. Returns an :class:`HTTPResponse` instance. - .. note:: - - Note that you must have read the whole response before you can send a new - request to the server. - .. versionchanged:: 3.5 If a :exc:`ConnectionError` or subclass is raised, the :class:`HTTPConnection` object will be ready to reconnect when a new request is sent. + Note that this does not apply to :exc:`OSError`\s raised by the underlying + socket. Instead the caller is responsible to call :meth:`close` on the + existing connection. + .. method:: HTTPConnection.set_debuglevel(level) @@ -429,7 +434,7 @@ HTTPConnection Objects The maximum number of allowed response headers to help prevent denial-of-service attacks. By default, the maximum number of allowed headers is set to 100. - .. versionadded:: next + .. versionadded:: 3.15 As an alternative to using the :meth:`~HTTPConnection.request` method described above, you can diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index 251aea891c3f989..5ee783b7fae950f 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -4,15 +4,12 @@ .. module:: http.cookiejar :synopsis: Classes for automatic handling of HTTP cookies. -.. moduleauthor:: John J. Lee -.. sectionauthor:: John J. Lee - **Source code:** :source:`Lib/http/cookiejar.py` -------------- -The :mod:`http.cookiejar` module defines classes for automatic handling of HTTP -cookies. It is useful for accessing web sites that require small pieces of data +The :mod:`!http.cookiejar` module defines classes for automatic handling of HTTP +cookies. It is useful for accessing websites that require small pieces of data -- :dfn:`cookies` -- to be set on the client machine by an HTTP response from a web server, and then returned to the server in later HTTP requests. @@ -21,7 +18,7 @@ Both the regular Netscape cookie protocol and the protocol defined by :rfc:`2109` cookies are parsed as Netscape cookies and subsequently treated either as Netscape or RFC 2965 cookies according to the 'policy' in effect. Note that the great majority of cookies on the internet are Netscape cookies. -:mod:`http.cookiejar` attempts to follow the de-facto Netscape cookie protocol (which +:mod:`!http.cookiejar` attempts to follow the de-facto Netscape cookie protocol (which differs substantially from that set out in the original Netscape specification), including taking note of the ``max-age`` and ``port`` cookie-attributes introduced with RFC 2965. @@ -109,7 +106,7 @@ The following classes are provided: .. class:: Cookie() This class represents Netscape, :rfc:`2109` and :rfc:`2965` cookies. It is not - expected that users of :mod:`http.cookiejar` construct their own :class:`Cookie` + expected that users of :mod:`!http.cookiejar` construct their own :class:`Cookie` instances. Instead, if necessary, call :meth:`make_cookies` on a :class:`CookieJar` instance. @@ -121,13 +118,13 @@ The following classes are provided: Module :mod:`http.cookies` HTTP cookie classes, principally useful for server-side code. The - :mod:`http.cookiejar` and :mod:`http.cookies` modules do not depend on each + :mod:`!http.cookiejar` and :mod:`http.cookies` modules do not depend on each other. https://curl.se/rfc/cookie_spec.html The specification of the original Netscape cookie protocol. Though this is still the dominant protocol, the 'Netscape cookie protocol' implemented by all - the major browsers (and :mod:`http.cookiejar`) only bears a passing resemblance to + the major browsers (and :mod:`!http.cookiejar`) only bears a passing resemblance to the one sketched out in ``cookie_spec.html``. :rfc:`2109` - HTTP State Management Mechanism @@ -617,7 +614,7 @@ standard cookie-attributes specified in the various cookie standards. The correspondence is not one-to-one, because there are complicated rules for assigning default values, because the ``max-age`` and ``expires`` cookie-attributes contain equivalent information, and because :rfc:`2109` cookies -may be 'downgraded' by :mod:`http.cookiejar` from version 1 to version 0 (Netscape) +may be 'downgraded' by :mod:`!http.cookiejar` from version 1 to version 0 (Netscape) cookies. Assignment to these attributes should not be necessary other than in rare @@ -629,7 +626,7 @@ internal consistency, so you should know what you're doing if you do that. Integer or :const:`None`. Netscape cookies have :attr:`version` 0. :rfc:`2965` and :rfc:`2109` cookies have a ``version`` cookie-attribute of 1. However, note that - :mod:`http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in which + :mod:`!http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in which case :attr:`version` is 0. @@ -692,7 +689,7 @@ internal consistency, so you should know what you're doing if you do that. ``True`` if this cookie was received as an :rfc:`2109` cookie (ie. the cookie arrived in a :mailheader:`Set-Cookie` header, and the value of the Version cookie-attribute in that header was 1). This attribute is provided because - :mod:`http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in + :mod:`!http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in which case :attr:`version` is 0. @@ -744,7 +741,7 @@ The :class:`Cookie` class also defines the following method: Examples -------- -The first example shows the most common usage of :mod:`http.cookiejar`:: +The first example shows the most common usage of :mod:`!http.cookiejar`:: import http.cookiejar, urllib.request cj = http.cookiejar.CookieJar() diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index eb1963207211945..4965c5fc3ba1d86 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -4,14 +4,11 @@ .. module:: http.cookies :synopsis: Support for HTTP state management (cookies). -.. moduleauthor:: Timothy O'Malley -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/http/cookies.py` -------------- -The :mod:`http.cookies` module defines classes for abstracting the concept of +The :mod:`!http.cookies` module defines classes for abstracting the concept of cookies, an HTTP state management mechanism. It supports both simple string-only cookies, and provides an abstraction for having any serializable data-type as cookie value. @@ -65,7 +62,7 @@ in a cookie name (as :attr:`~Morsel.key`). Module :mod:`http.cookiejar` HTTP cookie handling for web *clients*. The :mod:`http.cookiejar` and - :mod:`http.cookies` modules do not depend on each other. + :mod:`!http.cookies` modules do not depend on each other. :rfc:`2109` - HTTP State Management Mechanism This is the state management specification implemented by this module. @@ -110,6 +107,12 @@ Cookie Objects The meaning for *attrs* is the same as in :meth:`output`. + .. deprecated-removed:: 3.15 3.19 + This method generates a JavaScript snippet to set cookies in the browser, + which is no longer considered a standard or recommended approach. + Use :meth:`~http.cookies.BaseCookie.output` instead to generate HTTP + headers. + .. method:: BaseCookie.load(rawdata) @@ -148,9 +151,12 @@ Morsel Objects in HTTP requests, and is not accessible through JavaScript. This is intended to mitigate some forms of cross-site scripting. - The attribute :attr:`samesite` specifies that the browser is not allowed to - send the cookie along with cross-site requests. This helps to mitigate CSRF - attacks. Valid values for this attribute are "Strict" and "Lax". + The attribute :attr:`samesite` controls when the browser sends the cookie with + cross-site requests. This helps to mitigate CSRF attacks. Valid values are + "Strict" (only sent with same-site requests), "Lax" (sent with same-site + requests and top-level navigations), and "None" (sent with same-site and + cross-site requests). When using "None", the "secure" attribute must also + be set, as required by modern browsers. The attribute :attr:`partitioned` indicates to user agents that these cross-site cookies *should* only be available in the same top-level context @@ -223,6 +229,12 @@ Morsel Objects The meaning for *attrs* is the same as in :meth:`output`. + .. deprecated-removed:: 3.15 3.19 + This method generates a JavaScript snippet to set cookies in the browser, + which is no longer considered a standard or recommended approach. + Use :meth:`~http.cookies.Morsel.output` instead to generate HTTP + headers. + .. method:: Morsel.OutputString(attrs=None) @@ -261,7 +273,7 @@ Morsel Objects Example ------- -The following example demonstrates how to use the :mod:`http.cookies` module. +The following example demonstrates how to use the :mod:`!http.cookies` module. .. doctest:: :options: +NORMALIZE_WHITESPACE @@ -289,9 +301,9 @@ The following example demonstrates how to use the :mod:`http.cookies` module. Set-Cookie: chips=ahoy Set-Cookie: vienna=finger >>> C = cookies.SimpleCookie() - >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=\\012;";') + >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=;";') >>> print(C) - Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=\012;" + Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=;" >>> C = cookies.SimpleCookie() >>> C["oreo"] = "doublestuff" >>> C["oreo"]["path"] = "/" diff --git a/Doc/library/http.rst b/Doc/library/http.rst index ce3fb9f81205026..43a801416e24f99 100644 --- a/Doc/library/http.rst +++ b/Doc/library/http.rst @@ -12,7 +12,7 @@ -------------- -:mod:`http` is a package that collects several modules for working with the +:mod:`!http` is a package that collects several modules for working with the HyperText Transfer Protocol: * :mod:`http.client` is a low-level HTTP protocol client; for high-level URL @@ -22,7 +22,7 @@ HyperText Transfer Protocol: * :mod:`http.cookiejar` provides persistence of cookies -The :mod:`http` module also defines the following enums that help you work with http related code: +The :mod:`!http` module also defines the following enums that help you work with http related code: .. class:: HTTPStatus @@ -139,7 +139,8 @@ equal to the constant name (i.e. ``http.HTTPStatus.OK`` is also available as .. versionchanged:: 3.13 Implemented RFC9110 naming for status constants. Old constant names are preserved for - backwards compatibility. + backwards compatibility: ``413 REQUEST_ENTITY_TOO_LARGE``, ``414 REQUEST_URI_TOO_LONG``, + ``416 REQUESTED_RANGE_NOT_SATISFIABLE`` and ``422 UNPROCESSABLE_ENTITY``. HTTP status category -------------------- diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 063344e02842584..c4b9173f9e34eb2 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -19,7 +19,7 @@ This module defines classes for implementing HTTP servers. .. warning:: - :mod:`http.server` is not recommended for production. It only implements + :mod:`!http.server` is not recommended for production. It only implements :ref:`basic security checks `. .. include:: ../includes/wasm-notavail.rst @@ -99,7 +99,7 @@ instantiation, of which this module provides three different variants: This class is used to handle the HTTP requests that arrive at the server. By itself, it cannot respond to any actual HTTP requests; it must be subclassed - to handle each request method (e.g. GET or POST). + to handle each request method (for example, ``'GET'`` or ``'POST'``). :class:`BaseHTTPRequestHandler` provides a number of class and instance variables, and methods for use by subclasses. @@ -154,7 +154,7 @@ instantiation, of which this module provides three different variants: variable. This instance parses and manages the headers in the HTTP request. The :func:`~http.client.parse_headers` function from :mod:`http.client` is used to parse the headers and it requires that the - HTTP request provide a valid :rfc:`2822` style header. + HTTP request provide a valid :rfc:`5322` style header. .. attribute:: rfile @@ -241,7 +241,7 @@ instantiation, of which this module provides three different variants: request header it responds back with a ``100 Continue`` followed by ``200 OK`` headers. This method can be overridden to raise an error if the server does not - want the client to continue. For e.g. server can choose to send ``417 + want the client to continue. For example, the server can choose to send ``417 Expectation Failed`` as a response header and ``return False``. .. versionadded:: 3.2 @@ -287,6 +287,8 @@ instantiation, of which this module provides three different variants: specifying its value. Note that, after the send_header calls are done, :meth:`end_headers` MUST BE called in order to complete the operation. + This method does not reject input containing CRLF sequences. + .. versionchanged:: 3.2 Headers are stored in an internal buffer. @@ -297,6 +299,8 @@ instantiation, of which this module provides three different variants: buffered and sent directly the output stream.If the *message* is not specified, the HTTP message corresponding the response *code* is sent. + This method does not reject *message* containing CRLF sequences. + .. versionadded:: 3.2 .. method:: end_headers() @@ -362,7 +366,8 @@ instantiation, of which this module provides three different variants: delays, it now always returns the IP address. -.. class:: SimpleHTTPRequestHandler(request, client_address, server, directory=None) +.. class:: SimpleHTTPRequestHandler(request, client_address, server, \ + *, directory=None, extra_response_headers=None) This class serves files from the directory *directory* and below, or the current directory if *directory* is not provided, directly @@ -374,6 +379,9 @@ instantiation, of which this module provides three different variants: .. versionchanged:: 3.9 The *directory* parameter accepts a :term:`path-like object`. + .. versionchanged:: 3.15 + Added *extra_response_headers* parameter. + A lot of the work, such as parsing the request, is done by the base class :class:`BaseHTTPRequestHandler`. This class implements the :func:`do_GET` and :func:`do_HEAD` functions. @@ -386,6 +394,14 @@ instantiation, of which this module provides three different variants: This will be ``"SimpleHTTP/" + __version__``, where ``__version__`` is defined at the module level. + .. attribute:: default_content_type + + Specifies the Content-Type header value sent when the MIME type + cannot be guessed from the file extension of the requested URL. + By default, it is set to ``'application/octet-stream'``. + + .. versionadded:: 3.15 + .. attribute:: extensions_map A dictionary mapping suffixes into MIME types, contains custom overrides @@ -396,6 +412,15 @@ instantiation, of which this module provides three different variants: This dictionary is no longer filled with the default system mappings, but only contains overrides. + .. attribute:: extra_response_headers + + A sequence of ``(name, value)`` pairs containing user-defined extra HTTP + response headers to add to each successful HTTP status 200 response. These + headers are not included in other status code responses. + + Headers that the server sends automatically such as ``Content-Type`` + will not be overwritten by :attr:`!extra_response_headers`. + The :class:`SimpleHTTPRequestHandler` class defines the following methods: .. method:: do_HEAD() @@ -428,6 +453,9 @@ instantiation, of which this module provides three different variants: followed by a ``'Content-Length:'`` header with the file's size and a ``'Last-Modified:'`` header with the file's modification time. + The instance attribute :attr:`extra_response_headers` is a sequence of + ``(name, value)`` pairs containing user-defined extra response headers. + Then follows a blank line signifying the end of the headers, and then the contents of the file are output. @@ -463,9 +491,11 @@ such as using different index file names by overriding the class attribute Command-line interface ---------------------- -:mod:`http.server` can also be invoked directly using the :option:`-m` +:mod:`!http.server` can also be invoked directly using the :option:`-m` switch of the interpreter. The following example illustrates how to serve -files relative to the current directory:: +files relative to the current directory: + +.. code-block:: bash python -m http.server [OPTIONS] [port] @@ -476,7 +506,9 @@ The following options are accepted: .. option:: port The server listens to port 8000 by default. The default can be overridden - by passing the desired port number as an argument:: + by passing the desired port number as an argument: + + .. code-block:: bash python -m http.server 9000 @@ -485,7 +517,9 @@ The following options are accepted: Specifies a specific address to which it should bind. Both IPv4 and IPv6 addresses are supported. By default, the server binds itself to all interfaces. For example, the following command causes the server to bind - to localhost only:: + to localhost only: + + .. code-block:: bash python -m http.server --bind 127.0.0.1 @@ -498,7 +532,9 @@ The following options are accepted: Specifies a directory to which it should serve the files. By default, the server uses the current directory. For example, the following command - uses a specific directory:: + uses a specific directory: + + .. code-block:: bash python -m http.server --directory /tmp/ @@ -508,15 +544,31 @@ The following options are accepted: Specifies the HTTP version to which the server is conformant. By default, the server is conformant to HTTP/1.0. For example, the following command - runs an HTTP/1.1 conformant server:: + runs an HTTP/1.1 conformant server: + + .. code-block:: bash python -m http.server --protocol HTTP/1.1 .. versionadded:: 3.11 +.. option:: --content-type + + Specifies the default Content-Type HTTP header used when the MIME type + cannot be guessed from the URL's file extension. By default, the server + uses ``'application/octet-stream'``: + + .. code-block:: bash + + python -m http.server --content-type text/html + + .. versionadded:: 3.15 + .. option:: --tls-cert - Specifies a TLS certificate chain for HTTPS connections:: + Specifies a TLS certificate chain for HTTPS connections: + + .. code-block:: bash python -m http.server --tls-cert fullchain.pem @@ -532,17 +584,28 @@ The following options are accepted: .. option:: --tls-password-file - Specifies the password file for password-protected private keys:: + Specifies the password file for password-protected private keys: + + .. code-block:: bash python -m http.server \ --tls-cert cert.pem \ --tls-key key.pem \ --tls-password-file password.txt - This option requires `--tls-cert`` to be specified. + This option requires ``--tls-cert`` to be specified. .. versionadded:: 3.14 +.. option:: -H, --header
+ + Specify an additional extra HTTP Response Header to send on successful HTTP + 200 responses. Can be used multiple times to send additional custom response + headers. Headers that are sent automatically by the server (for instance + Content-Type) will not be overwritten by the server. + + .. versionadded:: 3.15 + .. _http.server-security: @@ -555,6 +618,11 @@ Security considerations requests, this makes it possible for files outside of the specified directory to be served. +Methods :meth:`BaseHTTPRequestHandler.send_header` and +:meth:`BaseHTTPRequestHandler.send_response_only` assume sanitized input +and do not perform input validation such as checking for the presence of CRLF +sequences. Untrusted input may result in HTTP Header injection attacks. + Earlier versions of Python did not scrub control characters from the log messages emitted to stderr from ``python -m http.server`` or the default :class:`BaseHTTPRequestHandler` ``.log_message`` diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index fabea611e0ebcd9..c7c30e5300c2a4a 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -3,8 +3,6 @@ IDLE --- Python editor and shell ================================ -.. moduleauthor:: Guido van Rossum - **Source code:** :source:`Lib/idlelib/` .. index:: @@ -13,7 +11,7 @@ IDLE --- Python editor and shell single: Integrated Development Environment .. - Remember to update Lib/idlelib/help.html with idlelib.help.copy_source() when modifying this file. + Remember to update Lib/idlelib/help.html with idlelib.help.copy_strip() when modifying this file. -------------- @@ -37,6 +35,10 @@ IDLE has the following features: * configuration, browsers, and other dialogs +The IDLE application is implemented in the :mod:`idlelib` package. + +.. include:: ../includes/optional-module.rst + Menus ----- @@ -88,7 +90,7 @@ Save Save As... Save the current window with a Save As dialog. The file saved becomes the - new associated file for the window. (If your file namager is set to hide + new associated file for the window. (If your file manager is set to hide extensions, the current extension will be omitted in the file name box. If the new filename has no '.', '.py' and '.txt' will be added for Python and text files, except that on macOS Aqua,'.py' is added for all files.) @@ -154,7 +156,7 @@ Go to Line Show Completions Open a scrollable list allowing selection of existing names. See - :ref:`Completions ` in the Editing and navigation section below. + :ref:`Completions ` in the Editing and Navigation section below. Expand Word Expand a prefix you have typed to match a full word in the same window; @@ -163,7 +165,7 @@ Expand Word Show Call Tip After an unclosed parenthesis for a function, open a small window with function parameter hints. See :ref:`Calltips ` in the - Editing and navigation section below. + Editing and Navigation section below. Show Surrounding Parens Highlight the surrounding parenthesis. @@ -174,9 +176,9 @@ Format menu (Editor window only) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Format Paragraph - Reformat the current blank-line-delimited paragraph in comment block or - multiline string or selected line in a string. All lines in the - paragraph will be formatted to less than N columns, where N defaults to 72. + Rewrap the text block containing the text insert cursor. + Avoid code lines. See :ref:`Format block` in the + Editing and Navigation section below. Indent Region Shift selected lines right by the indent width (default 4 spaces). @@ -204,9 +206,9 @@ New Indent Width Open a dialog to change indent width. The accepted default by the Python community is 4 spaces. -Strip Trailing Chitespace +Strip Trailing Whitespace Remove trailing space and other whitespace characters after the last - non-whitespace character of a line by applying str.rstrip to each line, + non-whitespace character of a line by applying :meth:`str.rstrip` to each line, including lines within multiline strings. Except for Shell windows, remove extra newlines at the end of the file. @@ -562,6 +564,20 @@ In an editor, import statements have no effect until one runs the file. One might want to run a file after writing import statements, after adding function definitions, or after opening an existing file. +.. _format-block: + +Format block +^^^^^^^^^^^^ + +Reformat Paragraph rewraps a block ('paragraph') of contiguous equally +indented non-blank comments, a similar block of text within a multiline +string, or a selected subset of either. +If needed, add a blank line to separate string from code. +Partial lines in a selection expand to complete lines. +The resulting lines have the same indent as before +but have maximum total length of N columns (characters). +Change the default N of 72 on the Window tab of IDLE Settings. + .. _code-context: Code Context @@ -657,7 +673,9 @@ looked for in the user's home directory. Statements in this file will be executed in the Tk namespace, so this file is not useful for importing functions to be used from IDLE's Python shell. -Command line usage +.. _idlelib-cli: + +Command-line usage ^^^^^^^^^^^^^^^^^^ .. program:: idle diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 9f198aebcb66b0b..df2468f7124e6d6 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -4,14 +4,6 @@ .. module:: imaplib :synopsis: IMAP4 protocol client (requires sockets). -.. moduleauthor:: Piers Lauder -.. sectionauthor:: Piers Lauder -.. revised by ESR, January 2000 -.. changes for IMAP4_SSL by Tino Lange , March 2002 -.. changes for IMAP4_stream by Piers Lauder , - November 2002 -.. changes for IMAP4 IDLE by Forest , August 2024 - **Source code:** :source:`Lib/imaplib.py` .. index:: @@ -29,7 +21,7 @@ note that the ``STATUS`` command is not supported in IMAP4. .. include:: ../includes/wasm-notavail.rst -Three classes are provided by the :mod:`imaplib` module, :class:`IMAP4` is the +Three classes are provided by the :mod:`!imaplib` module, :class:`IMAP4` is the base class: @@ -207,6 +199,11 @@ An :class:`IMAP4` instance has the following methods: Append *message* to named mailbox. + *flags* may be ``None`` or a string of IMAP flag tokens. Multiple + flags are separated by spaces, for example ``r'\Seen \Answered'``. + If *flags* is not already enclosed in parentheses, parentheses are + added automatically. + .. method:: IMAP4.authenticate(mechanism, authobject) @@ -413,6 +410,9 @@ An :class:`IMAP4` instance has the following methods: the password. Will only work if the server ``CAPABILITY`` response includes the phrase ``AUTH=CRAM-MD5``. + .. versionchanged:: 3.15 + An :exc:`IMAP4.error` is raised if MD5 support is not available. + .. method:: IMAP4.logout() @@ -700,6 +700,16 @@ The following attributes are defined on instances of :class:`IMAP4`: .. versionadded:: 3.5 +.. property:: IMAP4.file + + Internal :class:`~io.BufferedReader` associated with the underlying socket. + This property is documented for legacy purposes but not part of the public + interface. The caller is responsible to ensure that the current file is + closed before changing it. + + .. deprecated-removed:: 3.15 3.19 + + .. _imap4-example: IMAP4 Example diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 12014309e26ec93..e11db37b9fad501 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -18,11 +18,9 @@ the metadata of an installed `Distribution Package `_\s, modules, if any). Built in part on Python's import system, this library -intends to replace similar functionality in the `entry point -API`_ and `metadata API`_ of ``pkg_resources``. Along with -:mod:`importlib.resources`, -this package can eliminate the need to use the older and less efficient -``pkg_resources`` package. +provides the entry point and metadata APIs that were previously +exposed by the now-removed ``pkg_resources`` package. Along with +:mod:`importlib.resources`, it supersedes ``pkg_resources``. ``importlib.metadata`` operates on third-party *distribution packages* installed into Python's ``site-packages`` directory via tools such as @@ -107,6 +105,13 @@ You can also get a :ref:`distribution's version number `, list its current Python environment. +.. exception:: MetadataNotFound + + Subclass of :class:`FileNotFoundError` raised when attempting to load metadata + from a distribution folder that is empty or otherwise does not contain a + metadata file. + + Functional API ============== @@ -125,8 +130,8 @@ Entry points :meth:`!select` method for comparison to the attributes of the individual entry point definitions. - Note: it is not currently possible to query for entry points based on - their :attr:`!EntryPoint.dist` attribute (as different :class:`!Distribution` + Note: to query for entry points based on :attr:`!EntryPoint.dist` attribute, + use :meth:`Distribution.entry_points` instead (as different :class:`Distribution` instances do not currently compare equal, even if they have the same attributes) .. class:: EntryPoints @@ -226,6 +231,9 @@ Distribution metadata Raises :exc:`PackageNotFoundError` if the named distribution package is not installed in the current Python environment. + Raises :exc:`MetadataNotFound` if a distribution package is + present but no METADATA file is present. + .. class:: PackageMetadata A concrete implementation of the @@ -254,6 +262,12 @@ all the metadata in a JSON-compatible form per :PEP:`566`:: The full set of available metadata is not described here. See the PyPA `Core metadata specification `_ for additional details. +.. versionchanged:: 3.15 + Previously and incidentally, if a METADATA file was missing from a distribution, an + empty ``PackageMetadata`` would be returned, indistinguishable from + an empty METADATA file. Now, a missing METADATA file triggers a + ``MetadataNotFound`` exception. + .. versionchanged:: 3.10 The ``Description`` is now included in the metadata when presented through the payload. Line continuation characters have been removed. @@ -291,7 +305,7 @@ Distribution files .. function:: files(distribution_name) Return the full set of files contained within the named - distribution package. + distribution package as :class:`PackagePath` instances. Raises :exc:`PackageNotFoundError` if the named distribution package is not installed in the current Python environment. @@ -304,12 +318,22 @@ Distribution files A :class:`pathlib.PurePath` derived object with additional ``dist``, ``size``, and ``hash`` properties corresponding to the distribution - package's installation metadata for that file. + package's installation metadata for that file, also: + + .. method:: locate() + + If possible, return the concrete :class:`SimplePath` allowing to access data, + or raise a :exc:`NotImplementedError` otherwise. + +.. class:: SimplePath + + A protocol representing a minimal subset of :class:`pathlib.Path` that allows to + check if it ``exists()``, to traverse using ``joinpath()`` and ``parent``, + and to retrieve data using ``read_text()`` and ``read_bytes()``. The :func:`!files` function takes a `Distribution Package `_ -name and returns all of the files installed by this distribution. Each file is reported -as a :class:`PackagePath` instance. For example:: +name and returns all of the files installed by this distribution. For example:: >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP >>> util # doctest: +SKIP @@ -402,6 +426,18 @@ function is not reliable with such installs. Distributions ============= +While the module level API described above is the most common and convenient usage, +all that information is accessible from the :class:`Distribution` class. +:class:`!Distribution` is an abstract object that represents the metadata for +a Python `Distribution Package `_. +Get the concrete :class:`!Distribution` subclass instance for an installed +distribution package by calling the :func:`distribution` function:: + + >>> from importlib.metadata import distribution # doctest: +SKIP + >>> dist = distribution('wheel') # doctest: +SKIP + >>> type(dist) # doctest: +SKIP + + .. function:: distribution(distribution_name) Return a :class:`Distribution` instance describing the named @@ -410,6 +446,14 @@ Distributions Raises :exc:`PackageNotFoundError` if the named distribution package is not installed in the current Python environment. +Thus, an alternative way to get e.g. the version number is through the +:attr:`Distribution.version` attribute:: + + >>> dist.version # doctest: +SKIP + '0.32.3' + +The same applies for :func:`entry_points` and :func:`files`. + .. class:: Distribution Details of an installed distribution package. @@ -418,43 +462,88 @@ Distributions equal, even if they relate to the same installed distribution and accordingly have the same attributes. -While the module level API described above is the most common and convenient usage, -you can get all of that information from the :class:`!Distribution` class. -:class:`!Distribution` is an abstract object that represents the metadata for -a Python `Distribution Package `_. -You can get the concrete :class:`!Distribution` subclass instance for an installed -distribution package by calling the :func:`distribution` function:: + .. staticmethod:: at(path) + .. classmethod:: from_name(name) - >>> from importlib.metadata import distribution # doctest: +SKIP - >>> dist = distribution('wheel') # doctest: +SKIP - >>> type(dist) # doctest: +SKIP - + Return a :class:`!Distribution` instance at the given path or + with the given name. -Thus, an alternative way to get the version number is through the -:class:`!Distribution` instance:: + .. classmethod:: discover(*, context=None, **kwargs) - >>> dist.version # doctest: +SKIP - '0.32.3' + Returns an iterable of :class:`!Distribution` instances for all packages + (see distribution-discovery_). -There are all kinds of additional metadata available on :class:`!Distribution` -instances:: + The optional argument *context* is a :class:`DistributionFinder.Context` + instance, used to modify the search for distributions. Alternatively, + *kwargs* may contain keyword arguments for constructing a new + :class:`!DistributionFinder.Context`. - >>> dist.metadata['Requires-Python'] # doctest: +SKIP - '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' - >>> dist.metadata['License'] # doctest: +SKIP - 'MIT' + .. attribute:: metadata + :type: PackageMetadata -For editable packages, an ``origin`` property may present :pep:`610` -metadata:: + Raises :exc:`MetadataNotFound` if the METADATA file is not present in + the distribution. - >>> dist.origin.url - 'file:///path/to/wheel-0.32.3.editable-py3-none-any.whl' + There are all kinds of additional metadata available on :class:`!Distribution` + instances as a :class:`PackageMetadata` instance:: -The full set of available metadata is not described here. -See the PyPA `Core metadata specification `_ for additional details. + >>> dist.metadata['Requires-Python'] # doctest: +SKIP + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + >>> dist.metadata['License'] # doctest: +SKIP + 'MIT' + + The full set of available metadata is not described here. + See the PyPA `Core metadata specification `_ for additional details. -.. versionadded:: 3.13 - The ``.origin`` property was added. + .. attribute:: name + :type: str + .. attribute:: requires + :type: list[str] + .. attribute:: version + :type: str + + A few metadata fields are also available as shortcut properties. + + .. versionadded:: 3.10 + + The ``name`` shortcut was added. + + .. attribute:: origin + + For editable packages, an ``origin`` property may present :pep:`610` + metadata (for non-editable packages, ``origin`` is :const:`None`):: + + >>> dist.origin.url + 'file:///path/to/wheel-0.32.3.editable-py3-none-any.whl' + + The ``origin`` object follows the `Direct URL Data Structure + `_. + + .. versionadded:: 3.13 + + .. attribute:: entry_points + :type: EntryPoints + + The entry points provided by this distribution package. + + .. attribute:: files + :type: list[PackagePath] | None + + All files contained in this distribution package. + Like :func:`files`, this returns :const:`None` if there are no records. + + The following two abstract methods need to be implemented when implementing-custom-providers_: + + .. method:: locate_file(path) + + Like :meth:`!PackagePath.locate`, return a :class:`SimplePath` for the given path. + Takes a :class:`os.PathLike` or a :class:`str`. + + .. method:: read_text(filename) + + A shortcut for ``distribution.locate_file(filename).read_text()``. + +.. _distribution-discovery: Distribution Discovery ====================== @@ -466,6 +555,61 @@ This metadata finder search defaults to ``sys.path``, but varies slightly in how - ``importlib.metadata`` does not honor :class:`bytes` objects on ``sys.path``. - ``importlib.metadata`` will incidentally honor :py:class:`pathlib.Path` objects on ``sys.path`` even though such values will be ignored for imports. +.. class:: DistributionFinder + + A :class:`~importlib.abc.MetaPathFinder` subclass capable of discovering + installed distributions. + + Custom providers should implement this interface in order to + supply metadata. + + .. class:: Context(**kwargs) + + A :class:`!Context` gives a custom provider a means to + solicit additional details from the callers of distribution discovery + functions like :func:`distributions` or :meth:`Distribution.discover` + beyond :attr:`!.name` and :attr:`!.path` when searching + for distributions. + + For example, a provider could expose suites of packages in either a + "public" or "private" ``realm``. A caller of distribution discovery + functions may wish to query only for distributions in a particular realm + and could call ``distributions(realm="private")`` to signal to the + custom provider to only include distributions from that + realm. + + Each :class:`!DistributionFinder` must expect any parameters and should + attempt to honor the canonical parameters defined below when + appropriate. + + See the section on :ref:`implementing-custom-providers` for more details. + + .. attribute:: name + + Specific name for which a distribution finder should match. + + A :attr:`!.name` of ``None`` matches all distributions. + + .. attribute:: path + + A property providing the sequence of directory paths that a + distribution finder should search. + + Typically refers to Python installed package paths such as + "site-packages" directories and defaults to :attr:`sys.path`. + + +.. function:: distributions(**kwargs) + + Returns an iterable of :class:`Distribution` instances for all packages. + + The *kwargs* argument may contain either a keyword argument ``context``, a + :class:`DistributionFinder.Context` instance, or pass keyword arguments for + constructing a new :class:`!DistributionFinder.Context`. The + :class:`!DistributionFinder.Context` is used to modify the search for + distributions. + +.. _implementing-custom-providers: Implementing Custom Providers ============================= @@ -493,7 +637,7 @@ interface expected of finders by Python's import system. ``importlib.metadata`` extends this protocol by looking for an optional ``find_distributions`` callable on the finders from :data:`sys.meta_path` and presents this extended interface as the -``DistributionFinder`` abstract base class, which defines this abstract +:class:`DistributionFinder` abstract base class, which defines this abstract method:: @abc.abstractmethod @@ -502,14 +646,16 @@ method:: loading the metadata for packages for the indicated ``context``. """ -The ``DistributionFinder.Context`` object provides ``.path`` and ``.name`` -properties indicating the path to search and name to match and may -supply other relevant context sought by the consumer. +The :class:`DistributionFinder.Context` object provides +:attr:`~DistributionFinder.Context.path` and +:attr:`~DistributionFinder.Context.name` properties indicating the path to +search and name to match and may supply other relevant context sought by the +consumer. In practice, to support finding distribution package metadata in locations other than the file system, subclass -``Distribution`` and implement the abstract methods. Then from -a custom finder, return instances of this derived ``Distribution`` in the +:class:`!Distribution` and implement the abstract methods. Then from +a custom finder, return instances of this derived :class:`!Distribution` in the ``find_distributions()`` method. Example @@ -529,7 +675,7 @@ Imagine a custom finder that loads Python modules from a database:: That importer now presumably provides importable modules from a database, but it provides no metadata or entry points. For this custom importer to provide metadata, it would also need to implement -``DistributionFinder``:: +:class:`DistributionFinder`:: from importlib.metadata import DistributionFinder @@ -586,9 +732,5 @@ packages served by the ``DatabaseImporter``, assuming that the ``.entry_points`` attributes. The ``DatabaseDistribution`` may also provide other metadata files, like -``RECORD`` (required for ``Distribution.files``) or override the -implementation of ``Distribution.files``. See the source for more inspiration. - - -.. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points -.. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api +``RECORD`` (required for :attr:`!Distribution.files`) or override the +implementation of :attr:`!Distribution.files`. See the source for more inspiration. diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index 8253a33f591a0bc..45979c5691270aa 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -63,11 +63,14 @@ If the resource does not concretely exist on the file system, raise :exc:`FileNotFoundError`. - .. method:: is_resource(name) + .. method:: is_resource(path) :abstractmethod: - Returns ``True`` if the named *name* is considered a resource. - :exc:`FileNotFoundError` is raised if *name* does not exist. + Returns ``True`` if the named *path* is considered a resource. + :exc:`FileNotFoundError` is raised if *path* does not exist. + + .. versionchanged:: 3.10 + The argument *name* was renamed to *path*. .. method:: contents() :abstractmethod: diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 7a11f4fe0690048..72db66f9f06f890 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -31,15 +31,13 @@ not** have to exist as physical files and directories on the file system: for example, a package and its resources can be imported from a zip file using :py:mod:`zipimport`. -.. note:: +.. warning:: + + :mod:`importlib.resources` follows the same security model as the built-in + :func:`open` function. Passing untrusted inputs to the functions + in this module is unsafe. - This module provides functionality similar to `pkg_resources - `_ `Basic - Resource Access - `_ - without the performance overhead of that package. This makes reading - resources included in packages easier, with more stable and consistent - semantics. +.. note:: The standalone backport of this module provides more information on `using importlib.resources @@ -73,12 +71,15 @@ for example, a package and its resources can be imported from a zip file using .. versionadded:: 3.9 .. versionchanged:: 3.12 - *package* parameter was renamed to *anchor*. *anchor* can now - be a non-package module and if omitted will default to the caller's - module. *package* is still accepted for compatibility but will raise - a :exc:`DeprecationWarning`. Consider passing the anchor positionally or - using ``importlib_resources >= 5.10`` for a compatible interface - on older Pythons. + *package* parameter was renamed to *anchor*. + *package* was still accepted, but deprecated. + + .. versionchanged:: 3.15 + *package* parameter was fully removed. *anchor* can now be a + non-package module and if omitted will default to the caller's module. + *package* is no longer accepted since Python 3.15. Consider passing the + anchor positionally or using ``importlib_resources >= 5.10`` for a + compatible interface on older Pythons. .. function:: as_file(traversable) @@ -239,7 +240,6 @@ For all the following functions: .. versionchanged:: 3.13 Multiple *path_names* are accepted. - *encoding* and *errors* must be given as keyword arguments. .. function:: is_resource(anchor, *path_names) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 4f374be778d6b32..0b76020eacc1da2 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -4,9 +4,6 @@ .. module:: importlib :synopsis: The implementation of the import machinery. -.. moduleauthor:: Brett Cannon -.. sectionauthor:: Brett Cannon - .. versionadded:: 3.1 **Source code:** :source:`Lib/importlib/__init__.py` @@ -17,7 +14,7 @@ Introduction ------------ -The purpose of the :mod:`importlib` package is three-fold. +The purpose of the :mod:`!importlib` package is three-fold. One is to provide the implementation of the :keyword:`import` statement (and thus, by extension, the @@ -124,6 +121,10 @@ Functions need to call :func:`invalidate_caches` in order for the new module to be noticed by the import system. + If the module cannot be imported, :func:`import_module` will raise + :exc:`ImportError` or an appropriate subclass like + :exc:`ModuleNotFoundError`. + .. versionchanged:: 3.3 Parent packages are automatically imported. @@ -211,8 +212,8 @@ Functions in unexpected behavior. It's recommended to use the :class:`threading.Lock` or other synchronization primitives for thread-safe module reloading. -:mod:`importlib.abc` -- Abstract base classes related to import ---------------------------------------------------------------- +:mod:`!importlib.abc` -- Abstract base classes related to import +---------------------------------------------------------------- .. module:: importlib.abc :synopsis: Abstract base classes related to import @@ -222,7 +223,7 @@ Functions -------------- -The :mod:`importlib.abc` module contains all of the core abstract base classes +The :mod:`!importlib.abc` module contains all of the core abstract base classes used by :keyword:`import`. Some subclasses of the core abstract base classes are also provided to help in implementing the core ABCs. @@ -271,6 +272,28 @@ ABC hierarchy:: .. versionchanged:: 3.4 Returns ``None`` when called instead of :data:`NotImplemented`. + .. method:: discover(parent=None) + + An optional method which searches for possible specs with given *parent* + module spec. If *parent* is *None*, :meth:`MetaPathFinder.discover` will + search for top-level modules. + + Returns an iterable of possible specs. + + Raises :exc:`ValueError` if *parent* is not a package module. + + .. warning:: + This method can potentially yield a very large number of objects, and + it may carry out IO operations when computing these values. + + Because of this, it will generally be desirable to compute the result + values on-the-fly, as they are needed. As such, the returned object is + only guaranteed to be an :class:`iterable `, + instead of a :class:`list` or other + :class:`collection ` type. + + .. versionadded:: 3.15 + .. class:: PathEntryFinder @@ -303,6 +326,28 @@ ABC hierarchy:: :meth:`importlib.machinery.PathFinder.invalidate_caches` when invalidating the caches of all cached finders. + .. method:: discover(parent=None) + + An optional method which searches for possible specs with given *parent* + module spec. If *parent* is *None*, :meth:`PathEntryFinder.discover` will + search for top-level modules. + + Returns an iterable of possible specs. + + Raises :exc:`ValueError` if *parent* is not a package module. + + .. warning:: + This method can potentially yield a very large number of objects, and + it may carry out IO operations when computing these values. + + Because of this, it will generally be desirable to compute the result + values on-the-fly, as they are needed. As such, the returned object is + only guaranteed to be an :class:`iterable `, + instead of a :class:`list` or other + :class:`collection ` type. + + .. versionadded:: 3.15 + .. class:: Loader @@ -316,6 +361,9 @@ ABC hierarchy:: .. versionchanged:: 3.7 Introduced the optional :meth:`get_resource_reader` method. + .. versionchanged:: 3.15 + Removed the ``load_module()`` method. + .. method:: create_module(spec) A method that returns the module object to use when @@ -340,47 +388,6 @@ ABC hierarchy:: .. versionchanged:: 3.6 :meth:`create_module` must also be defined. - .. method:: load_module(fullname) - - A legacy method for loading a module. If the module cannot be - loaded, :exc:`ImportError` is raised, otherwise the loaded module is - returned. - - If the requested module already exists in :data:`sys.modules`, that - module should be used and reloaded. - Otherwise the loader should create a new module and insert it into - :data:`sys.modules` before any loading begins, to prevent recursion - from the import. If the loader inserted a module and the load fails, it - must be removed by the loader from :data:`sys.modules`; modules already - in :data:`sys.modules` before the loader began execution should be left - alone. - - The loader should set several attributes on the module - (note that some of these attributes can change when a module is - reloaded): - - - :attr:`module.__name__` - - :attr:`module.__file__` - - :attr:`module.__cached__` *(deprecated)* - - :attr:`module.__path__` - - :attr:`module.__package__` *(deprecated)* - - :attr:`module.__loader__` *(deprecated)* - - When :meth:`exec_module` is available then backwards-compatible - functionality is provided. - - .. versionchanged:: 3.4 - Raise :exc:`ImportError` when called instead of - :exc:`NotImplementedError`. Functionality provided when - :meth:`exec_module` is available. - - .. deprecated-removed:: 3.4 3.15 - The recommended API for loading a module is :meth:`exec_module` - (and :meth:`create_module`). Loaders should implement it instead of - :meth:`load_module`. The import machinery takes care of all the - other responsibilities of :meth:`load_module` when - :meth:`exec_module` is implemented. - .. class:: ResourceLoader @@ -393,6 +400,8 @@ ABC hierarchy:: .. deprecated:: 3.7 This ABC is deprecated in favour of supporting resource loading through :class:`importlib.resources.abc.TraversableResources`. + This class exists for backwards compatibility only with other ABCs in + this module. .. method:: get_data(path) :abstractmethod: @@ -453,7 +462,7 @@ ABC hierarchy:: .. versionchanged:: 3.4 Raises :exc:`ImportError` instead of :exc:`NotImplementedError`. - .. staticmethod:: source_to_code(data, path='') + .. staticmethod:: source_to_code(data, path='', fullname=None) Create a code object from Python source. @@ -465,24 +474,25 @@ ABC hierarchy:: With the subsequent code object one can execute it in a module by running ``exec(code, module.__dict__)``. + The optional argument *fullname* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax + warnings by module name. + .. versionadded:: 3.4 .. versionchanged:: 3.5 Made the method static. + .. versionadded:: 3.15 + Added the *fullname* parameter. + + .. method:: exec_module(module) Implementation of :meth:`Loader.exec_module`. .. versionadded:: 3.4 - .. method:: load_module(fullname) - - Implementation of :meth:`Loader.load_module`. - - .. deprecated-removed:: 3.4 3.15 - use :meth:`exec_module` instead. - .. class:: ExecutionLoader @@ -516,6 +526,9 @@ ABC hierarchy:: .. versionadded:: 3.3 + .. versionchanged:: 3.15 + Removed the ``load_module()`` method. + .. attribute:: name The name of the module the loader can handle. @@ -524,13 +537,6 @@ ABC hierarchy:: Path to the file of the module. - .. method:: load_module(fullname) - - Calls super's ``load_module()``. - - .. deprecated-removed:: 3.4 3.15 - Use :meth:`Loader.exec_module` instead. - .. method:: get_filename(fullname) :abstractmethod: @@ -562,6 +568,9 @@ ABC hierarchy:: optimization to speed up loading by removing the parsing step of Python's compiler, and so no bytecode-specific API is exposed. + .. versionchanged:: 3.15 + Removed the ``load_module()`` method. + .. method:: path_stats(path) Optional abstract method which returns a :class:`dict` containing @@ -615,13 +624,6 @@ ABC hierarchy:: .. versionadded:: 3.4 - .. method:: load_module(fullname) - - Concrete implementation of :meth:`Loader.load_module`. - - .. deprecated-removed:: 3.4 3.15 - Use :meth:`exec_module` instead. - .. method:: get_source(fullname) Concrete implementation of :meth:`InspectLoader.get_source`. @@ -635,174 +637,8 @@ ABC hierarchy:: itself does not end in ``__init__``. -.. class:: ResourceReader - - *Superseded by TraversableResources* - - An :term:`abstract base class` to provide the ability to read - *resources*. - - From the perspective of this ABC, a *resource* is a binary - artifact that is shipped within a package. Typically this is - something like a data file that lives next to the ``__init__.py`` - file of the package. The purpose of this class is to help abstract - out the accessing of such data files so that it does not matter if - the package and its data file(s) are stored e.g. in a zip file - versus on the file system. - - For any of methods of this class, a *resource* argument is - expected to be a :term:`path-like object` which represents - conceptually just a file name. This means that no subdirectory - paths should be included in the *resource* argument. This is - because the location of the package the reader is for, acts as the - "directory". Hence the metaphor for directories and file - names is packages and resources, respectively. This is also why - instances of this class are expected to directly correlate to - a specific package (instead of potentially representing multiple - packages or a module). - - Loaders that wish to support resource reading are expected to - provide a method called ``get_resource_reader(fullname)`` which - returns an object implementing this ABC's interface. If the module - specified by fullname is not a package, this method should return - :const:`None`. An object compatible with this ABC should only be - returned when the specified module is a package. - - .. versionadded:: 3.7 - - .. deprecated-removed:: 3.12 3.14 - Use :class:`importlib.resources.abc.TraversableResources` instead. - - .. method:: open_resource(resource) - :abstractmethod: - - Returns an opened, :term:`file-like object` for binary reading - of the *resource*. - - If the resource cannot be found, :exc:`FileNotFoundError` is - raised. - - .. method:: resource_path(resource) - :abstractmethod: - - Returns the file system path to the *resource*. - - If the resource does not concretely exist on the file system, - raise :exc:`FileNotFoundError`. - - .. method:: is_resource(name) - :abstractmethod: - - Returns ``True`` if the named *name* is considered a resource. - :exc:`FileNotFoundError` is raised if *name* does not exist. - - .. method:: contents() - :abstractmethod: - - Returns an :term:`iterable` of strings over the contents of - the package. Do note that it is not required that all names - returned by the iterator be actual resources, e.g. it is - acceptable to return names for which :meth:`is_resource` would - be false. - - Allowing non-resource names to be returned is to allow for - situations where how a package and its resources are stored - are known a priori and the non-resource names would be useful. - For instance, returning subdirectory names is allowed so that - when it is known that the package and resources are stored on - the file system then those subdirectory names can be used - directly. - - The abstract method returns an iterable of no items. - - -.. class:: Traversable - - An object with a subset of :class:`pathlib.Path` methods suitable for - traversing directories and opening files. - - For a representation of the object on the file-system, use - :meth:`importlib.resources.as_file`. - - .. versionadded:: 3.9 - - .. deprecated-removed:: 3.12 3.14 - Use :class:`importlib.resources.abc.Traversable` instead. - - .. attribute:: name - - Abstract. The base name of this object without any parent references. - - .. method:: iterdir() - :abstractmethod: - - Yield ``Traversable`` objects in ``self``. - - .. method:: is_dir() - :abstractmethod: - - Return ``True`` if ``self`` is a directory. - - .. method:: is_file() - :abstractmethod: - - Return ``True`` if ``self`` is a file. - - .. method:: joinpath(child) - :abstractmethod: - - Return Traversable child in ``self``. - - .. method:: __truediv__(child) - :abstractmethod: - - Return ``Traversable`` child in ``self``. - - .. method:: open(mode='r', *args, **kwargs) - :abstractmethod: - - *mode* may be 'r' or 'rb' to open as text or binary. Return a handle - suitable for reading (same as :attr:`pathlib.Path.open`). - - When opening as text, accepts encoding parameters such as those - accepted by :class:`io.TextIOWrapper`. - - .. method:: read_bytes() - - Read contents of ``self`` as bytes. - - .. method:: read_text(encoding=None) - - Read contents of ``self`` as text. - - -.. class:: TraversableResources - - An abstract base class for resource readers capable of serving - the :meth:`importlib.resources.files` interface. Subclasses - :class:`importlib.resources.abc.ResourceReader` and provides - concrete implementations of the :class:`importlib.resources.abc.ResourceReader`'s - abstract methods. Therefore, any loader supplying - :class:`importlib.abc.TraversableResources` also supplies ResourceReader. - - Loaders that wish to support resource reading are expected to - implement this interface. - - .. versionadded:: 3.9 - - .. deprecated-removed:: 3.12 3.14 - Use :class:`importlib.resources.abc.TraversableResources` instead. - - .. method:: files() - :abstractmethod: - - Returns a :class:`importlib.resources.abc.Traversable` object for the loaded - package. - - - -:mod:`importlib.machinery` -- Importers and path hooks ------------------------------------------------------- +:mod:`!importlib.machinery` -- Importers and path hooks +------------------------------------------------------- .. module:: importlib.machinery :synopsis: Importers and path hooks @@ -1007,6 +843,36 @@ find and load modules. :exc:`ImportError` is raised. +.. class:: NamespacePath(name, path, path_finder) + + Represents a :term:`namespace package`'s path (:attr:`module.__path__`). + + When its ``__path__`` value is accessed it will be recomputed if necessary. + This keeps it in-sync with the global state (:attr:`sys.modules`). + + The *name* argument is the name of the namespace module. + + The *path* argument is the initial path value. + + The *path_finder* argument is the callable used to recompute the path value. + The callable has the same signature as :meth:`importlib.abc.MetaPathFinder.find_spec`. + + When the parent's :attr:`module.__path__` attribute is updated, the path + value is recomputed. + + If the parent module is missing from :data:`sys.modules`, then + :exc:`ModuleNotFoundError` will be raised. + + For top-level modules, the parent module's path is :data:`sys.path`. + + .. note:: + + :meth:`PathFinder.invalidate_caches` invalidates :class:`NamespacePath`, + forcing the path value to be recomputed next time it is accessed. + + .. versionadded:: 3.15 + + .. class:: SourceFileLoader(fullname, path) A concrete implementation of :class:`importlib.abc.SourceLoader` by @@ -1015,6 +881,9 @@ find and load modules. .. versionadded:: 3.3 + .. versionchanged:: 3.15 + Removed the ``load_module()`` method. + .. attribute:: name The name of the module that this loader will handle. @@ -1035,15 +904,6 @@ find and load modules. Concrete implementation of :meth:`importlib.abc.SourceLoader.set_data`. - .. method:: load_module(name=None) - - Concrete implementation of :meth:`importlib.abc.Loader.load_module` where - specifying the name of the module to load is optional. - - .. deprecated-removed:: 3.6 3.15 - - Use :meth:`importlib.abc.Loader.exec_module` instead. - .. class:: SourcelessFileLoader(fullname, path) @@ -1057,6 +917,9 @@ find and load modules. .. versionadded:: 3.3 + .. versionchanged:: 3.15 + Removed the ``load_module()`` method. + .. attribute:: name The name of the module the loader will handle. @@ -1078,15 +941,6 @@ find and load modules. Returns ``None`` as bytecode files have no source when this loader is used. - .. method:: load_module(name=None) - - Concrete implementation of :meth:`importlib.abc.Loader.load_module` where - specifying the name of the module to load is optional. - - .. deprecated-removed:: 3.6 3.15 - - Use :meth:`importlib.abc.Loader.exec_module` instead. - .. class:: ExtensionFileLoader(fullname, path) @@ -1218,8 +1072,7 @@ find and load modules. .. attribute:: cached - The filename of a compiled version of the module's code - (see :attr:`module.__cached__`). + The filename of a compiled version of the module's code. The :term:`finder` should always set this attribute but it may be ``None`` for modules that do not need compiled code stored. @@ -1251,7 +1104,7 @@ find and load modules. To accommodate this requirement, when running on iOS, extension module binaries are *not* packaged as ``.so`` files on ``sys.path``, but as individual standalone frameworks. To discover those frameworks, this loader - is be registered against the ``.fwork`` file extension, with a ``.fwork`` + is registered against the ``.fwork`` file extension, with a ``.fwork`` file acting as a placeholder in the original location of the binary on ``sys.path``. The ``.fwork`` file contains the path of the actual binary in the ``Frameworks`` folder, relative to the app bundle. To allow for @@ -1300,8 +1153,8 @@ find and load modules. Path to the ``.fwork`` file for the extension module. -:mod:`importlib.util` -- Utility code for importers ---------------------------------------------------- +:mod:`!importlib.util` -- Utility code for importers +---------------------------------------------------- .. module:: importlib.util :synopsis: Utility code for importers @@ -1321,7 +1174,7 @@ an :term:`importer`. .. versionadded:: 3.4 -.. function:: cache_from_source(path, debug_override=None, *, optimization=None) +.. function:: cache_from_source(path, *, optimization=None) Return the :pep:`3147`/:pep:`488` path to the byte-compiled file associated with the source *path*. For example, if *path* is ``/foo/bar/baz.py`` the return @@ -1340,12 +1193,6 @@ an :term:`importer`. ``/foo/bar/__pycache__/baz.cpython-32.opt-2.pyc``. The string representation of *optimization* can only be alphanumeric, else :exc:`ValueError` is raised. - The *debug_override* parameter is deprecated and can be used to override - the system's value for ``__debug__``. A ``True`` value is the equivalent of - setting *optimization* to the empty string. A ``False`` value is the same as - setting *optimization* to ``1``. If both *debug_override* an *optimization* - are not ``None`` then :exc:`TypeError` is raised. - .. versionadded:: 3.4 .. versionchanged:: 3.5 @@ -1355,6 +1202,9 @@ an :term:`importer`. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.15 + The *debug_override* parameter was removed. + .. function:: source_from_cache(path) diff --git a/Doc/library/index.rst b/Doc/library/index.rst index 44b218948d07e1a..8fc77be520d4268 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -43,6 +43,7 @@ the `Python Package Index `_. constants.rst stdtypes.rst exceptions.rst + threadsafety.rst text.rst binary.rst @@ -63,7 +64,6 @@ the `Python Package Index `_. internet.rst mm.rst i18n.rst - frameworks.rst tk.rst development.rst debug.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index e8d1176f4778667..a0f7379b12a8a62 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -9,14 +9,11 @@ .. module:: inspect :synopsis: Extract information and source code from live objects. -.. moduleauthor:: Ka-Ping Yee -.. sectionauthor:: Ka-Ping Yee - **Source code:** :source:`Lib/inspect.py` -------------- -The :mod:`inspect` module provides several useful functions to help get +The :mod:`!inspect` module provides several useful functions to help get information about live objects such as modules, classes, methods, functions, tracebacks, frame objects, and code objects. For example, it can help you examine the contents of a class, retrieve the source code of a method, extract @@ -198,10 +195,6 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | | read more :ref:`here | | | | `| +-----------------+-------------------+---------------------------+ -| | co_lnotab | encoded mapping of line | -| | | numbers to bytecode | -| | | indices | -+-----------------+-------------------+---------------------------+ | | co_freevars | tuple of names of free | | | | variables (referenced via | | | | a function's closure) | @@ -253,12 +246,21 @@ attributes (see :ref:`import-mod-attrs` for module attributes): +-----------------+-------------------+---------------------------+ | | gi_running | is the generator running? | +-----------------+-------------------+---------------------------+ +| | gi_suspended | is the generator | +| | | suspended? | ++-----------------+-------------------+---------------------------+ | | gi_code | code | +-----------------+-------------------+---------------------------+ | | gi_yieldfrom | object being iterated by | | | | ``yield from``, or | | | | ``None`` | +-----------------+-------------------+---------------------------+ +| | gi_state | state of the generator, | +| | | one of ``GEN_CREATED``, | +| | | ``GEN_RUNNING``, | +| | | ``GEN_SUSPENDED``, or | +| | | ``GEN_CLOSED`` | ++-----------------+-------------------+---------------------------+ | async generator | __name__ | name | +-----------------+-------------------+---------------------------+ | | __qualname__ | qualified name | @@ -270,8 +272,18 @@ attributes (see :ref:`import-mod-attrs` for module attributes): +-----------------+-------------------+---------------------------+ | | ag_running | is the generator running? | +-----------------+-------------------+---------------------------+ +| | ag_suspended | is the generator | +| | | suspended? | ++-----------------+-------------------+---------------------------+ | | ag_code | code | +-----------------+-------------------+---------------------------+ +| | ag_state | state of the async | +| | | generator, one of | +| | | ``AGEN_CREATED``, | +| | | ``AGEN_RUNNING``, | +| | | ``AGEN_SUSPENDED``, or | +| | | ``AGEN_CLOSED`` | ++-----------------+-------------------+---------------------------+ | coroutine | __name__ | name | +-----------------+-------------------+---------------------------+ | | __qualname__ | qualified name | @@ -283,12 +295,21 @@ attributes (see :ref:`import-mod-attrs` for module attributes): +-----------------+-------------------+---------------------------+ | | cr_running | is the coroutine running? | +-----------------+-------------------+---------------------------+ +| | cr_suspended | is the coroutine | +| | | suspended? | ++-----------------+-------------------+---------------------------+ | | cr_code | code | +-----------------+-------------------+---------------------------+ | | cr_origin | where coroutine was | | | | created, or ``None``. See | | | | |coroutine-origin-link| | +-----------------+-------------------+---------------------------+ +| | cr_state | state of the coroutine, | +| | | one of ``CORO_CREATED``, | +| | | ``CORO_RUNNING``, | +| | | ``CORO_SUSPENDED``, or | +| | | ``CORO_CLOSED`` | ++-----------------+-------------------+---------------------------+ | builtin | __doc__ | documentation string | +-----------------+-------------------+---------------------------+ | | __name__ | original name of this | @@ -316,10 +337,27 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Add ``__builtins__`` attribute to functions. +.. versionchanged:: 3.11 + + Add ``gi_suspended`` attribute to generators. + +.. versionchanged:: 3.11 + + Add ``cr_suspended`` attribute to coroutines. + +.. versionchanged:: 3.12 + + Add ``ag_suspended`` attribute to async generators. + .. versionchanged:: 3.14 Add ``f_generator`` attribute to frames. +.. versionchanged:: 3.15 + + Add ``gi_state`` attribute to generators, ``cr_state`` attribute to + coroutines, and ``ag_state`` attribute to async generators. + .. function:: getmembers(object[, predicate]) Return all the members of an object in a list of ``(name, value)`` @@ -378,17 +416,47 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a class, whether built-in or created in Python code. + This function returns ``False`` for :ref:`generic aliases ` of classes, + such as ``list[int]``. + .. function:: ismethod(object) Return ``True`` if the object is a bound method written in Python. + .. note:: -.. function:: ispackage(object) + For example, given this class:: - Return ``True`` if the object is a :term:`package`. + >>> class Greeter: + ... def say_hello(self): + ... print('hello!') - .. versionadded:: 3.14 + A bound method (also known as an *instance method*) is created when + accessing ``say_hello`` (a :term:`function` defined in the + ``Greeter`` namespace) through an instance of the ``Greeter`` class:: + + >>> instance = Greeter() + + >>> instance.say_hello + > + >>> ismethod(instance.say_hello) + True + >>> isfunction(instance.say_hello) + False + + Accessing ``say_hello`` through the ``Greeter`` class will return the + function itself. For this function, :func:`ismethod` will return + ``False``, but :func:`isfunction` will return ``True``:: + + >>> Greeter.say_hello + + >>> ismethod(Greeter.say_hello) + False + >>> isfunction(Greeter.say_hello) + True + + See :ref:`typesmethods` for details. .. function:: isfunction(object) @@ -396,11 +464,23 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a Python function, which includes functions created by a :term:`lambda` expression. + See the note for :func:`~inspect.ismethod` for an example. + + +.. function:: ispackage(object) + + Return ``True`` if the object is a :term:`package`. + + .. versionadded:: 3.14 + .. function:: isgeneratorfunction(object) Return ``True`` if the object is a Python generator function. + It also returns ``True`` for bound methods created from Python generator functions + (see :ref:`typesmethods` for more information). + .. versionchanged:: 3.8 Functions wrapped in :func:`functools.partial` now return ``True`` if the wrapped function is a Python generator function. @@ -503,7 +583,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. versionchanged:: 3.13 Functions wrapped in :func:`functools.partialmethod` now return ``True`` - if the wrapped function is a :term:`coroutine function`. + if the wrapped function is a :term:`asynchronous generator` function. .. function:: isasyncgen(object) @@ -616,17 +696,29 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Retrieving source code ---------------------- -.. function:: getdoc(object) +.. function:: getdoc(object, *, inherit_class_doc=True, fallback_to_class_doc=True) Get the documentation string for an object, cleaned up with :func:`cleandoc`. - If the documentation string for an object is not provided and the object is - a class, a method, a property or a descriptor, retrieve the documentation - string from the inheritance hierarchy. + If the documentation string for an object is not provided: + + * if the object is a class and *inherit_class_doc* is true (by default), + retrieve the documentation string from the inheritance hierarchy; + * if the object is a method, a property or a descriptor, retrieve + the documentation string from the inheritance hierarchy; + * otherwise, if *fallback_to_class_doc* is true (by default), retrieve + the documentation string from the class of the object. + Return ``None`` if the documentation string is invalid or missing. .. versionchanged:: 3.5 Documentation strings are now inherited if not overridden. + .. versionchanged:: 3.15 + Added parameters *inherit_class_doc* and *fallback_to_class_doc*. + + Documentation strings on :class:`~functools.cached_property` + objects are now inherited if not overridden. + .. function:: getcomments(object) @@ -1139,7 +1231,7 @@ Classes and functions times. -.. function:: getfullargspec(func) +.. function:: getfullargspec(func, *, annotation_format=Format.VALUE) Get the names and default values of a Python function's parameters. A :term:`named tuple` is returned: @@ -1169,6 +1261,14 @@ Classes and functions APIs. This function is retained primarily for use in code that needs to maintain compatibility with the Python 2 ``inspect`` module API. + A member of the + :class:`annotationlib.Format` enum can be passed to the + *annotation_format* parameter to control the format of the returned + annotations. For example, use + ``annotation_format=annotationlib.Format.STRING`` to return annotations in string + format. Note that with the default ``VALUE`` format, creation of some argspecs + may raise an exception. + .. versionchanged:: 3.4 This function is now based on :func:`signature`, but still ignores ``__wrapped__`` attributes and includes the already bound first @@ -1179,13 +1279,16 @@ Classes and functions :func:`signature` in Python 3.5, but that decision has been reversed in order to restore a clearly supported standard interface for single-source Python 2/3 code migrating away from the legacy - :func:`getargspec` API. + :func:`!getargspec` API. .. versionchanged:: 3.7 Python only explicitly guaranteed that it preserved the declaration order of keyword-only parameters as of version 3.7, although in practice this order had always been preserved in Python 3. + .. versionchanged:: 3.15 + The *annotation_format* parameter was added. + .. function:: getargvalues(frame) @@ -1289,6 +1392,11 @@ Classes and functions This is an alias for :func:`annotationlib.get_annotations`; see the documentation of that function for more information. + .. caution:: + + This function may execute arbitrary code contained in annotations. + See :ref:`annotationlib-security` for more information. + .. versionadded:: 3.10 .. versionchanged:: 3.14 @@ -1506,10 +1614,11 @@ properties, will be invoked and :meth:`~object.__getattr__` and may be called. For cases where you want passive introspection, like documentation tools, this -can be inconvenient. :func:`getattr_static` has the same signature as :func:`getattr` +can be inconvenient. :func:`getattr_static` has a similar signature as :func:`getattr` but avoids executing code when it fetches attributes. -.. function:: getattr_static(obj, attr, default=None) +.. function:: getattr_static(obj, attr) + getattr_static(obj, attr, default) Retrieve attributes without triggering dynamic lookup via the descriptor protocol, :meth:`~object.__getattr__` @@ -1729,7 +1838,7 @@ which is a bitmap of the following flags: The flags are specific to CPython, and may not be defined in other Python implementations. Furthermore, the flags are an implementation detail, and can be removed or deprecated in future Python releases. - It's recommended to use public APIs from the :mod:`inspect` module + It's recommended to use public APIs from the :mod:`!inspect` module for any introspection needs. @@ -1768,18 +1877,25 @@ Buffer flags .. _inspect-module-cli: -Command Line Interface +Command-line interface ---------------------- -The :mod:`inspect` module also provides a basic introspection capability +The :mod:`!inspect` module also provides a basic introspection capability from the command line. .. program:: inspect By default, accepts the name of a module and prints the source of that module. A class or function within the module can be printed instead by -appended a colon and the qualified name of the target object. +appending a colon and the qualified name of the target object. .. option:: --details Print information about the specified object rather than the source code + +.. versionchanged:: 3.15 + + The ``--details`` option now supports basic introspection for modules + without available source code and indicates when modules are frozen. + It also indicates when the given target reference is not the canonical + name of the referenced object. diff --git a/Doc/library/io.rst b/Doc/library/io.rst index dfebccb5a9cb91c..d47b74efe22de9d 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -4,14 +4,6 @@ .. module:: io :synopsis: Core tools for working with streams. -.. moduleauthor:: Guido van Rossum -.. moduleauthor:: Mike Verdone -.. moduleauthor:: Mark Russell -.. moduleauthor:: Antoine Pitrou -.. moduleauthor:: Amaury Forgeot d'Arc -.. moduleauthor:: Benjamin Peterson -.. sectionauthor:: Benjamin Peterson - **Source code:** :source:`Lib/io.py` -------------- @@ -24,7 +16,7 @@ Overview .. index:: single: file object; io module -The :mod:`io` module provides Python's main facilities for dealing with various +The :mod:`!io` module provides Python's main facilities for dealing with various types of I/O. There are three main types of I/O: *text I/O*, *binary I/O* and *raw I/O*. These are generic categories, and various backing stores can be used for each of them. A concrete object belonging to any of these @@ -46,6 +38,7 @@ will raise a :exc:`TypeError`. So will giving a :class:`bytes` object to the Operations that used to raise :exc:`IOError` now raise :exc:`OSError`, since :exc:`IOError` is now an alias of :exc:`OSError`. +.. _text-io: Text I/O ^^^^^^^^ @@ -73,6 +66,7 @@ In-memory text streams are also available as :class:`StringIO` objects:: The text stream API is described in detail in the documentation of :class:`TextIOBase`. +.. _binary-io: Binary I/O ^^^^^^^^^^ @@ -111,6 +105,13 @@ stream by opening a file in binary mode with buffering disabled:: The raw stream API is described in detail in the docs of :class:`RawIOBase`. +.. warning:: + Raw I/O is a low-level interface and methods generally must have their return + values checked and be explicitly retried to ensure an operation completes. + For instance :meth:`~RawIOBase.write` returns the number of bytes written + which may be less than the number of bytes provided (a partial write). + High-level I/O objects like :ref:`binary-io` and :ref:`text-io` implement + retry behavior. .. _io-text-encoding: @@ -292,7 +293,7 @@ interface to a buffered raw stream (:class:`BufferedIOBase`). Finally, Argument names are not part of the specification, and only the arguments of :func:`open` are intended to be used as keyword arguments. -The following table summarizes the ABCs provided by the :mod:`io` module: +The following table summarizes the ABCs provided by the :mod:`!io` module: .. tabularcolumns:: |l|l|L|L| @@ -486,8 +487,11 @@ I/O Base Classes Read up to *size* bytes from the object and return them. As a convenience, if *size* is unspecified or -1, all bytes until EOF are returned. - Otherwise, only one system call is ever made. Fewer than *size* bytes may - be returned if the operating system call returns fewer than *size* bytes. + + Attempts to make only one system call but will retry if interrupted and + the signal handler does not raise an exception (see :pep:`475` for the + rationale). This means fewer than *size* bytes may be returned if the + operating system call returns fewer than *size* bytes. If 0 bytes are returned, and *size* was not 0, this indicates end of file. If the object is in non-blocking mode and no bytes are available, @@ -501,13 +505,19 @@ I/O Base Classes Read and return all the bytes from the stream until EOF, using multiple calls to the stream if necessary. + If ``0`` bytes are returned this indicates end of file. If the object is in + non-blocking mode and the underlying :meth:`read` returns ``None`` + indicating no bytes are available, ``None`` is returned. + .. method:: readinto(b, /) Read bytes into a pre-allocated, writable :term:`bytes-like object` *b*, and return the number of bytes read. For example, *b* might be a :class:`bytearray`. - If the object is in non-blocking mode and no bytes - are available, ``None`` is returned. + + If ``0`` is returned and ``len(b)`` is not ``0``, this indicates end of file. If + the object is in non-blocking mode and no bytes are available, ``None`` is + returned. .. method:: write(b, /) @@ -521,6 +531,13 @@ I/O Base Classes this method returns, so the implementation should only access *b* during the method call. + .. warning:: + + This function does not ensure all bytes are written or an exception is + thrown. Callers may implement that behavior by checking the return + value and, if it is less than the length of *b*, looping with additional + write calls until all unwritten bytes are written. High-level I/O + objects like :ref:`binary-io` and :ref:`text-io` implement retry behavior. .. class:: BufferedIOBase @@ -587,7 +604,7 @@ I/O Base Classes When the underlying raw stream is non-blocking, implementations may either raise :exc:`BlockingIOError` or return ``None`` if no data is - available. :mod:`io` implementations return ``None``. + available. :mod:`!io` implementations return ``None``. .. method:: read1(size=-1, /) @@ -600,7 +617,7 @@ I/O Base Classes When the underlying raw stream is non-blocking, implementations may either raise :exc:`BlockingIOError` or return ``None`` if no data is - available. :mod:`io` implementations return ``None``. + available. :mod:`!io` implementations return ``None``. .. method:: readinto(b, /) @@ -649,7 +666,11 @@ Raw File I/O .. class:: FileIO(name, mode='r', closefd=True, opener=None) A raw binary stream representing an OS-level file containing bytes data. It - inherits from :class:`RawIOBase`. + inherits from :class:`RawIOBase` and implements its low-level access design. + This means :meth:`~RawIOBase.write` does not guarantee all bytes are written + and :meth:`~RawIOBase.read` may read less bytes than requested even when more + bytes may be present in the underlying file. To get "write all" and + "read at least" behavior, use :ref:`binary-io`. The *name* can be one of two things: @@ -669,10 +690,6 @@ Raw File I/O implies writing, so this mode behaves in a similar way to ``'w'``. Add a ``'+'`` to the mode to allow simultaneous reading and writing. - The :meth:`~RawIOBase.read` (when called with a positive argument), - :meth:`~RawIOBase.readinto` and :meth:`~RawIOBase.write` methods on this - class will only make one system call. - A custom opener can be used by passing a callable as *opener*. The underlying file descriptor for the file object is then obtained by calling *opener* with (*name*, *flags*). *opener* must return an open file descriptor (passing @@ -684,6 +701,13 @@ Raw File I/O See the :func:`open` built-in function for examples on using the *opener* parameter. + .. warning:: + :class:`FileIO` is a low-level I/O object and members, such as + :meth:`~RawIOBase.read` and :meth:`~RawIOBase.write`, need to have their + return values checked explicitly in a retry loop to implement "write all" + and "read at least" behavior. High-level I/O objects :ref:`binary-io` and + :ref:`text-io` implement retry behavior. + .. versionchanged:: 3.3 The *opener* parameter was added. The ``'x'`` mode was added. @@ -720,7 +744,7 @@ than raw I/O does. contains initial data. Methods may be used from multiple threads without external locking in - :term:`free threading` builds. + :term:`free-threaded builds `. :class:`BytesIO` provides or overrides these methods in addition to those from :class:`BufferedIOBase` and :class:`IOBase`: @@ -832,9 +856,9 @@ than raw I/O does. .. class:: BufferedRandom(raw, buffer_size=DEFAULT_BUFFER_SIZE) - A buffered binary stream providing higher-level access to a seekable - :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedReader` - and :class:`BufferedWriter`. + A buffered binary stream implementing :class:`BufferedIOBase` interfaces + providing higher-level access to a seekable :class:`RawIOBase` raw binary + stream. The constructor creates a reader and writer for a seekable raw stream, given in the first argument. If the *buffer_size* is omitted it defaults to diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 9e887d8e65741b2..9ccd8602bcb2c33 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -4,13 +4,11 @@ .. module:: ipaddress :synopsis: IPv4/IPv6 manipulation library. -.. moduleauthor:: Peter Moody - **Source code:** :source:`Lib/ipaddress.py` -------------- -:mod:`ipaddress` provides the capabilities to create, manipulate and +:mod:`!ipaddress` provides the capabilities to create, manipulate and operate on IPv4 and IPv6 addresses and networks. The functions and classes in this module make it straightforward to handle @@ -34,7 +32,7 @@ This is the full module API reference—for an overview and introduction, see Convenience factory functions ----------------------------- -The :mod:`ipaddress` module provides factory functions to conveniently create +The :mod:`!ipaddress` module provides factory functions to conveniently create IP addresses, networks and interfaces: .. function:: ip_address(address) @@ -369,9 +367,9 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: ipv4_mapped - For addresses that appear to be IPv4 mapped addresses (starting with - ``::FFFF/96``), this property will report the embedded IPv4 address. - For any other address, this property will be ``None``. + For addresses that appear to be IPv4 mapped addresses in the range + ``::FFFF:0:0/96`` as defined by :RFC:`4291`, this property reports the + embedded IPv4 address. For any other address, this property will be ``None``. .. attribute:: scope_id @@ -1027,7 +1025,7 @@ The module also provides the following module level functions: IPv4Address('192.0.2.0') <= IPv4Network('192.0.2.0/24') doesn't make sense. There are some times however, where you may wish to - have :mod:`ipaddress` sort these anyway. If you need to do this, you can use + have :mod:`!ipaddress` sort these anyway. If you need to do this, you can use this function as the *key* argument to :func:`sorted`. *obj* is either a network or address object. diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index aa46920d3526f0f..06f8bf2a8b6fa81 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -4,9 +4,6 @@ .. module:: itertools :synopsis: Functions creating iterators for efficient looping. -.. moduleauthor:: Raymond Hettinger -.. sectionauthor:: Raymond Hettinger - .. testsetup:: from itertools import * @@ -30,32 +27,24 @@ For instance, SML provides a tabulation tool: ``tabulate(f)`` which produces a sequence ``f(0), f(1), ...``. The same effect can be achieved in Python by combining :func:`map` and :func:`count` to form ``map(f, count())``. - -**Infinite iterators:** - -================== ================= ================================================= ========================================= -Iterator Arguments Results Example -================== ================= ================================================= ========================================= -:func:`count` [start[, step]] start, start+step, start+2*step, ... ``count(10) → 10 11 12 13 14 ...`` -:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') → A B C D A B C D ...`` -:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) → 10 10 10`` -================== ================= ================================================= ========================================= - -**Iterators terminating on the shortest input sequence:** +**General iterators:** ============================ ============================ ================================================= ============================================================= Iterator Arguments Results Example ============================ ============================ ================================================= ============================================================= :func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) → 1 3 6 10 15`` -:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=2) → AB CD EF G`` +:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=3) → ABC DEF G`` :func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') → A B C D E F`` :func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) → A B C D E F`` :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) → A C E F`` +:func:`count` [start[, step]] start, start+step, start+2*step, ... ``count(10) → 10 11 12 13 14 ...`` +:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') → A B C D A B C D ...`` :func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8`` :func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8`` :func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) ``groupby(['A','B','DEF'], len) → (1, A B) (3, DEF)`` :func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) → C D E F G`` :func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') → AB BC CD DE EF FG`` +:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) → 10 10 10`` :func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000`` :func:`takewhile` predicate, seq seq[0], seq[1], until predicate fails ``takewhile(lambda x: x<5, [1,4,6,3,8]) → 1 4`` :func:`tee` it, n it1, it2, ... itn splits one iterator into n ``tee('ABC', 2) → A B C, A B C`` @@ -181,7 +170,7 @@ loops that truncate the stream. Roughly equivalent to:: def batched(iterable, n, *, strict=False): - # batched('ABCDEFG', 2) → AB CD EF G + # batched('ABCDEFG', 3) → ABC DEF G if n < 1: raise ValueError('n must be at least one') iterator = iter(iterable) @@ -819,7 +808,7 @@ well as with the built-in itertools such as ``map()``, ``filter()``, A secondary purpose of the recipes is to serve as an incubator. The ``accumulate()``, ``compress()``, and ``pairwise()`` itertools started out as -recipes. Currently, the ``sliding_window()``, ``iter_index()``, and ``sieve()`` +recipes. Currently, the ``sliding_window()``, ``derangements()``, and ``sieve()`` recipes are being tested to see whether they prove their worth. Substantially all of these recipes and many, many others can be installed from @@ -838,11 +827,18 @@ and :term:`generators ` which incur interpreter overhead. .. testcode:: + from itertools import (accumulate, batched, chain, combinations, compress, + count, cycle, filterfalse, groupby, islice, permutations, product, + repeat, starmap, tee, zip_longest) from collections import Counter, deque from contextlib import suppress from functools import reduce - from math import comb, prod, sumprod, isqrt - from operator import itemgetter, getitem, mul, neg + from heapq import heappush, heappushpop, heappush_max, heappushpop_max + from math import comb, isqrt, prod, sumprod + from operator import getitem, is_not, itemgetter, mul, neg, truediv + + + # ==== Basic one liners ==== def take(n, iterable): "Return first n items of the iterable as a list." @@ -853,10 +849,6 @@ and :term:`generators ` which incur interpreter overhead. # prepend(1, [2, 3, 4]) → 1 2 3 4 return chain([value], iterable) - def tabulate(function, start=0): - "Return function(0), function(1), ..." - return map(function, count(start)) - def repeatfunc(function, times=None, *args): "Repeat calls to a function with specified arguments." if times is None: @@ -899,8 +891,8 @@ and :term:`generators ` which incur interpreter overhead. def first_true(iterable, default=False, predicate=None): "Returns the first true value or the *default* if there is no true value." - # first_true([a,b,c], x) → a or b or c or x - # first_true([a,b], x, f) → a if f(a) else b if f(b) else x + # first_true([a, b, c], x) → a or b or c or x + # first_true([a, b], x, f) → a if f(a) else b if f(b) else x return next(filter(predicate, iterable), default) def all_equal(iterable, key=None): @@ -908,6 +900,9 @@ and :term:`generators ` which incur interpreter overhead. # all_equal('4٤௪౪໔', key=int) → True return len(take(2, groupby(iterable, key))) <= 1 + + # ==== Data pipelines ==== + def unique_justseen(iterable, key=None): "Yield unique elements, preserving order. Remember only the element just seen." # unique_justseen('AAAABBBCCDAABBB') → A B C D A B @@ -933,14 +928,14 @@ and :term:`generators ` which incur interpreter overhead. yield element def unique(iterable, key=None, reverse=False): - "Yield unique elements in sorted order. Supports unhashable inputs." - # unique([[1, 2], [3, 4], [1, 2]]) → [1, 2] [3, 4] - sequenced = sorted(iterable, key=key, reverse=reverse) - return unique_justseen(sequenced, key=key) + "Yield unique elements in sorted order. Supports unhashable inputs." + # unique([[1, 2], [3, 4], [1, 2]]) → [1, 2] [3, 4] + sequenced = sorted(iterable, key=key, reverse=reverse) + return unique_justseen(sequenced, key=key) def sliding_window(iterable, n): "Collect data into overlapping fixed-length chunks or blocks." - # sliding_window('ABCDEFG', 4) → ABCD BCDE CDEF DEFG + # sliding_window('ABCDEFG', 3) → ABC BCD CDE DEF EFG iterator = iter(iterable) window = deque(islice(iterator, n - 1), maxlen=n) for x in iterator: @@ -949,7 +944,7 @@ and :term:`generators ` which incur interpreter overhead. def grouper(iterable, n, *, incomplete='fill', fillvalue=None): "Collect data into non-overlapping fixed-length chunks or blocks." - # grouper('ABCDEFG', 3, fillvalue='x') → ABC DEF Gxx + # grouper('ABCDEFG', 3, fillvalue='x') → ABC DEF Gxx # grouper('ABCDEFG', 3, incomplete='strict') → ABC DEF ValueError # grouper('ABCDEFG', 3, incomplete='ignore') → ABC DEF iterators = [iter(iterable)] * n @@ -978,6 +973,16 @@ and :term:`generators ` which incur interpreter overhead. slices = starmap(slice, combinations(range(len(seq) + 1), 2)) return map(getitem, repeat(seq), slices) + def derangements(iterable, r=None): + "Produce r length permutations without fixed points." + # derangements('ABCD') → BADC BCDA BDAC CADB CDAB CDBA DABC DCAB DCBA + # Algorithm credited to Stefan Pochmann + seq = tuple(iterable) + pos = tuple(range(len(seq))) + have_moved = map(map, repeat(is_not), repeat(pos), permutations(pos, r=r)) + valid_derangements = map(all, have_moved) + return compress(permutations(seq, r=r), valid_derangements) + def iter_index(iterable, value, start=0, stop=None): "Return indices where a value occurs in a sequence or iterable." # iter_index('AABCADEAF', 'A') → 0 1 4 7 @@ -1005,9 +1010,7 @@ and :term:`generators ` which incur interpreter overhead. yield function() -The following recipes have a more mathematical flavor: - -.. testcode:: + # ==== Mathematical operations ==== def multinomial(*counts): "Number of distinct arrangements of a multiset." @@ -1026,9 +1029,12 @@ The following recipes have a more mathematical flavor: # sum_of_squares([10, 20, 30]) → 1400 return sumprod(*tee(iterable)) + + # ==== Matrix operations ==== + def reshape(matrix, columns): "Reshape a 2-D matrix to have a given number of columns." - # reshape([(0, 1), (2, 3), (4, 5)], 3) → (0, 1, 2), (3, 4, 5) + # reshape([(0, 1), (2, 3), (4, 5)], 3) → (0, 1, 2) (3, 4, 5) return batched(chain.from_iterable(matrix), columns, strict=True) def transpose(matrix): @@ -1038,10 +1044,13 @@ The following recipes have a more mathematical flavor: def matmul(m1, m2): "Multiply two matrices." - # matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)]) → (49, 80), (41, 60) + # matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)]) → (49, 80) (41, 60) n = len(m2[0]) return batched(starmap(sumprod, product(m1, transpose(m2))), n) + + # ==== Polynomial arithmetic ==== + def convolve(signal, kernel): """Discrete linear convolution of two iterables. Equivalent to polynomial multiplication. @@ -1096,6 +1105,9 @@ The following recipes have a more mathematical flavor: powers = reversed(range(1, n)) return list(map(mul, coefficients, powers)) + + # ==== Number theory ==== + def sieve(n): "Primes less than n." # sieve(30) → 2 3 5 7 11 13 17 19 23 29 @@ -1134,6 +1146,49 @@ The following recipes have a more mathematical flavor: return n + # ==== Running statistics ==== + + def running_mean(iterable): + "Average of values seen so far." + # running_mean([37, 33, 38, 28]) → 37 35 36 34 + return map(truediv, accumulate(iterable), count(1)) + + def running_min(iterable): + "Smallest of values seen so far." + # running_min([37, 33, 38, 28]) → 37 33 33 28 + return accumulate(iterable, func=min) + + def running_max(iterable): + "Largest of values seen so far." + # running_max([37, 33, 38, 28]) → 37 37 38 38 + return accumulate(iterable, func=max) + + def running_median(iterable): + "Median of values seen so far." + # running_median([37, 33, 38, 28]) → 37 35 37 35 + read = iter(iterable).__next__ + lo = [] # max-heap + hi = [] # min-heap the same size as or one smaller than lo + with suppress(StopIteration): + while True: + heappush_max(lo, heappushpop(hi, read())) + yield lo[0] + heappush(hi, heappushpop_max(lo, read())) + yield (lo[0] + hi[0]) / 2 + + def running_statistics(iterable): + "Aggregate statistics for values seen so far." + # Generate tuples: (size, minimum, median, maximum, mean) + t0, t1, t2, t3 = tee(iterable, 4) + return zip( + count(1), + running_min(t0), + running_median(t1), + running_max(t2), + running_mean(t3), + ) + + .. doctest:: :hide: @@ -1210,10 +1265,6 @@ The following recipes have a more mathematical flavor: [(0, 'a'), (1, 'b'), (2, 'c')] - >>> list(islice(tabulate(lambda x: 2*x), 4)) - [0, 2, 4, 6] - - >>> for _ in loops(5): ... print('hi') ... @@ -1663,6 +1714,36 @@ The following recipes have a more mathematical flavor: ['A', 'AB', 'ABC', 'ABCD', 'B', 'BC', 'BCD', 'C', 'CD', 'D'] + >>> ' '.join(map(''.join, derangements('ABCD'))) + 'BADC BCDA BDAC CADB CDAB CDBA DABC DCAB DCBA' + >>> ' '.join(map(''.join, derangements('ABCD', 3))) + 'BAD BCA BCD BDA CAB CAD CDA CDB DAB DCA DCB' + >>> ' '.join(map(''.join, derangements('ABCD', 2))) + 'BA BC BD CA CD DA DC' + >>> ' '.join(map(''.join, derangements('ABCD', 1))) + 'B C D' + >>> ' '.join(map(''.join, derangements('ABCD', 0))) + '' + >>> # Compare number of derangements to https://oeis.org/A000166 + >>> [len(list(derangements(range(n)))) for n in range(10)] + [1, 0, 1, 2, 9, 44, 265, 1854, 14833, 133496] + >>> # Verify that identical objects are treated as unique by position + >>> identical = 'X' + >>> distinct = 'x' + >>> seq1 = ('A', identical, 'B', identical) + >>> result1 = ' '.join(map(''.join, derangements(seq1))) + >>> result1 + 'XAXB XBXA XXAB BAXX BXAX BXXA XAXB XBAX XBXA' + >>> seq2 = ('A', identical, 'B', distinct) + >>> result2 = ' '.join(map(''.join, derangements(seq2))) + >>> result2 + 'XAxB XBxA XxAB BAxX BxAX BxXA xAXB xBAX xBXA' + >>> result1 == result2 + False + >>> result1.casefold() == result2.casefold() + True + + >>> list(powerset([1,2,3])) [(), (1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)] >>> all(len(list(powerset(range(n)))) == 2**n for n in range(18)) @@ -1743,11 +1824,37 @@ The following recipes have a more mathematical flavor: True + >>> list(running_mean([8.5, 9.5, 7.5, 6.5])) + [8.5, 9.0, 8.5, 8.0] + >>> list(running_mean([37, 33, 38, 28])) + [37.0, 35.0, 36.0, 34.0] + + + >>> list(running_min([37, 33, 38, 28])) + [37, 33, 33, 28] + + + >>> list(running_max([37, 33, 38, 28])) + [37, 37, 38, 38] + + + >>> list(running_median([37, 33, 38, 28])) + [37, 35.0, 37, 35.0] + + + >>> list(running_statistics([37, 33, 38, 28])) + [(1, 37, 37, 37, 37.0), (2, 33, 35.0, 37, 35.0), (3, 33, 37, 38, 36.0), (4, 28, 35.0, 38, 34.0)] + + .. testcode:: :hide: # Old recipes and their tests which are guaranteed to continue to work. + def tabulate(function, start=0): + "Return function(0), function(1), ..." + return map(function, count(start)) + def old_sumprod_recipe(vec1, vec2): "Compute a sum of products." return sum(starmap(operator.mul, zip(vec1, vec2, strict=True))) @@ -1827,6 +1934,10 @@ The following recipes have a more mathematical flavor: .. doctest:: :hide: + >>> list(islice(tabulate(lambda x: 2*x), 4)) + [0, 2, 4, 6] + + >>> dotproduct([1,2,3], [4,5,6]) 32 diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 12a5a96a3c56f3b..383ccad9df041b5 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -4,9 +4,6 @@ .. module:: json :synopsis: Encode and decode the JSON format. -.. moduleauthor:: Bob Ippolito -.. sectionauthor:: Bob Ippolito - **Source code:** :source:`Lib/json/__init__.py` -------------- @@ -121,7 +118,7 @@ Extending :class:`JSONEncoder`:: ['[2.0', ', 1.0', ']'] -Using :mod:`json` from the shell to validate and pretty-print: +Using :mod:`!json` from the shell to validate and pretty-print: .. code-block:: shell-session @@ -183,8 +180,10 @@ Basic Usage :param bool ensure_ascii: If ``True`` (the default), the output is guaranteed to - have all incoming non-ASCII characters escaped. - If ``False``, these characters will be outputted as-is. + have all incoming non-ASCII and non-printable characters escaped. + If ``False``, all characters will be outputted as-is, except for + the characters that must be escaped: quotation mark, reverse solidus, + and the control characters U+0000 through U+001F. :param bool check_circular: If ``False``, the circular reference check for container types is skipped @@ -212,7 +211,7 @@ Basic Usage a string (such as ``"\t"``) is used to indent each level. If zero, negative, or ``""`` (the empty string), only newlines are inserted. - If ``None`` (the default), the most compact representation is used. + If ``None`` (the default), no newlines are inserted. :type indent: int | str | None :param separators: @@ -265,7 +264,7 @@ Basic Usage .. function:: load(fp, *, cls=None, object_hook=None, parse_float=None, \ parse_int=None, parse_constant=None, \ - object_pairs_hook=None, **kw) + object_pairs_hook=None, array_hook=None, **kw) Deserialize *fp* to a Python object using the :ref:`JSON-to-Python conversion table `. @@ -302,6 +301,15 @@ Basic Usage Default ``None``. :type object_pairs_hook: :term:`callable` | None + :param array_hook: + If set, a function that is called with the result of + any JSON array literal decoded with as a Python list. + The return value of this function will be used + instead of the :class:`list`. + This feature can be used to implement custom decoders. + Default ``None``. + :type array_hook: :term:`callable` | None + :param parse_float: If set, a function that is called with the string of every JSON float to be decoded. @@ -350,7 +358,10 @@ Basic Usage conversion length limitation ` to help avoid denial of service attacks. -.. function:: loads(s, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) + .. versionchanged:: 3.15 + Added the optional *array_hook* parameter. + +.. function:: loads(s, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, array_hook=None, **kw) Identical to :func:`load`, but instead of a file-like object, deserialize *s* (a :class:`str`, :class:`bytes` or :class:`bytearray` @@ -368,7 +379,7 @@ Basic Usage Encoders and Decoders --------------------- -.. class:: JSONDecoder(*, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, strict=True, object_pairs_hook=None) +.. class:: JSONDecoder(*, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, strict=True, object_pairs_hook=None, array_hook=None) Simple JSON decoder. @@ -413,6 +424,14 @@ Encoders and Decoders .. versionchanged:: 3.1 Added support for *object_pairs_hook*. + *array_hook* is an optional function that will be called with the + result of every JSON array decoded as a list. The return value of + *array_hook* will be used instead of the :class:`list`. This feature can be + used to implement custom decoders. + + .. versionchanged:: 3.15 + Added support for *array_hook*. + *parse_float* is an optional function that will be called with the string of every JSON float to be decoded. By default, this is equivalent to ``float(num_str)``. This can be used to use another datatype or parser for @@ -495,8 +514,10 @@ Encoders and Decoders :class:`bool` or ``None``. If *skipkeys* is true, such items are simply skipped. If *ensure_ascii* is true (the default), the output is guaranteed to - have all incoming non-ASCII characters escaped. If *ensure_ascii* is - false, these characters will be output as-is. + have all incoming non-ASCII and non-printable characters escaped. + If *ensure_ascii* is false, all characters will be output as-is, except for + the characters that must be escaped: quotation mark, reverse solidus, + and the control characters U+0000 through U+001F. If *check_circular* is true (the default), then lists, dicts, and custom encoded objects will be checked for circular references during encoding to @@ -636,7 +657,7 @@ UTF-32, with UTF-8 being the recommended default for maximum interoperability. As permitted, though not required, by the RFC, this module's serializer sets *ensure_ascii=True* by default, thus escaping the output so that the resulting -strings only contain ASCII characters. +strings only contain printable ASCII characters. Other than the *ensure_ascii* parameter, this module is defined strictly in terms of conversion between Python objects and @@ -743,8 +764,8 @@ Command-line interface -------------- -The :mod:`json` module can be invoked as a script via ``python -m json`` -to validate and pretty-print JSON objects. The :mod:`json.tool` submodule +The :mod:`!json` module can be invoked as a script via ``python -m json`` +to validate and pretty-print JSON objects. The :mod:`!json.tool` submodule implements this interface. If the optional ``infile`` and ``outfile`` arguments are not @@ -765,7 +786,7 @@ specified, :data:`sys.stdin` and :data:`sys.stdout` will be used respectively: alphabetically by key. .. versionchanged:: 3.14 - The :mod:`json` module may now be directly executed as + The :mod:`!json` module may now be directly executed as ``python -m json``. For backwards compatibility, invoking the CLI as ``python -m json.tool`` remains supported. diff --git a/Doc/library/linecache.rst b/Doc/library/linecache.rst index e766a9280946d30..ff07499e58f0f6d 100644 --- a/Doc/library/linecache.rst +++ b/Doc/library/linecache.rst @@ -4,13 +4,11 @@ .. module:: linecache :synopsis: Provides random access to individual lines from text files. -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/linecache.py` -------------- -The :mod:`linecache` module allows one to get any line from a Python source file, while +The :mod:`!linecache` module allows one to get any line from a Python source file, while attempting to optimize internally, using a cache, the common case where many lines are read from a single file. This is used by the :mod:`traceback` module to retrieve source lines for inclusion in the formatted traceback. @@ -19,7 +17,7 @@ The :func:`tokenize.open` function is used to open files. This function uses :func:`tokenize.detect_encoding` to get the encoding of the file; in the absence of an encoding token, the file encoding defaults to UTF-8. -The :mod:`linecache` module defines the following functions: +The :mod:`!linecache` module defines the following functions: .. function:: getline(filename, lineno, module_globals=None) @@ -31,7 +29,7 @@ The :mod:`linecache` module defines the following functions: .. index:: triple: module; search; path If *filename* indicates a frozen module (starting with ``' -.. sectionauthor:: Martin von Löwis - **Source code:** :source:`Lib/locale.py` -------------- -The :mod:`locale` module opens access to the POSIX locale database and +The :mod:`!locale` module opens access to the POSIX locale database and functionality. The POSIX locale mechanism allows programmers to deal with certain cultural issues in an application, without requiring the programmer to know all the specifics of each country where the software is executed. .. index:: pair: module; _locale -The :mod:`locale` module is implemented on top of the :mod:`!_locale` module, +The :mod:`!locale` module is implemented on top of the :mod:`!_locale` module, which in turn uses an ANSI C locale implementation if available. -The :mod:`locale` module defines the following exception and functions: +The :mod:`!locale` module defines the following exception and functions: .. exception:: Error @@ -34,17 +31,33 @@ The :mod:`locale` module defines the following exception and functions: If *locale* is given and not ``None``, :func:`setlocale` modifies the locale setting for the *category*. The available categories are listed in the data - description below. *locale* may be a string, or an iterable of two strings - (language code and encoding). If it's an iterable, it's converted to a locale - name using the locale aliasing engine. An empty string specifies the user's + description below. *locale* may be a :ref:`string `, or a pair, + language code and encoding. An empty string specifies the user's default settings. If the modification of the locale fails, the exception :exc:`Error` is raised. If successful, the new locale setting is returned. + If *locale* is a pair, it is converted to a locale name using + the locale aliasing engine. + The language code has the same format as a :ref:`locale name `, + but without encoding. + The language code and encoding can be ``None``. + If *locale* is omitted or ``None``, the current setting for *category* is returned. + Example:: + + >>> import locale + >>> loc = locale.setlocale(locale.LC_ALL) # get current locale + # use German locale; name and availability varies with platform + >>> locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8') + >>> locale.strcoll('f\xe4n', 'foo') # compare a string containing an umlaut + >>> locale.setlocale(locale.LC_ALL, '') # use user's preferred locale + >>> locale.setlocale(locale.LC_ALL, 'C') # use default (C) locale + >>> locale.setlocale(locale.LC_ALL, loc) # restore saved locale + :func:`setlocale` is not thread-safe on most systems. Applications typically - start with a call of :: + start with a call of:: import locale locale.setlocale(locale.LC_ALL, '') @@ -53,6 +66,9 @@ The :mod:`locale` module defines the following exception and functions: specified in the :envvar:`LANG` environment variable). If the locale is not changed thereafter, using multithreading should not cause problems. + .. versionchanged:: 3.15 + Support language codes with ``@``-modifiers. + .. function:: localeconv() @@ -345,22 +361,28 @@ The :mod:`locale` module defines the following exception and functions: ``'LANG'``. The GNU gettext search path contains ``'LC_ALL'``, ``'LC_CTYPE'``, ``'LANG'`` and ``'LANGUAGE'``, in that order. - Except for the code ``'C'``, the language code corresponds to :rfc:`1766`. - *language code* and *encoding* may be ``None`` if their values cannot be + The language code has the same format as a :ref:`locale name `, + but without encoding and ``@``-modifier. + The language code and encoding may be ``None`` if their values cannot be determined. - - .. deprecated-removed:: 3.11 3.15 + The "C" locale is represented as ``(None, None)``. .. function:: getlocale(category=LC_CTYPE) - Returns the current setting for the given locale category as sequence containing - *language code*, *encoding*. *category* may be one of the :const:`!LC_\*` values - except :const:`LC_ALL`. It defaults to :const:`LC_CTYPE`. + Returns the current setting for the given locale category as a tuple containing + the language code and encoding. *category* may be one of the :const:`!LC_\*` + values except :const:`LC_ALL`. It defaults to :const:`LC_CTYPE`. - Except for the code ``'C'``, the language code corresponds to :rfc:`1766`. - *language code* and *encoding* may be ``None`` if their values cannot be + The language code has the same format as a :ref:`locale name `, + but without encoding. + The language code and encoding may be ``None`` if their values cannot be determined. + The "C" locale is represented as ``(None, None)``. + + .. versionchanged:: 3.15 + ``@``-modifier are no longer silently removed, but included in + the language code. .. function:: getpreferredencoding(do_setlocale=True) @@ -508,14 +530,14 @@ The :mod:`locale` module defines the following exception and functions: SSH connections. Python doesn't internally use locale-dependent character transformation functions - from ``ctype.h``. Instead, an internal ``pyctype.h`` provides locale-independent - equivalents like :c:macro:`!Py_TOLOWER`. + from ``ctype.h``. Instead, ``pyctype.h`` provides locale-independent + equivalents like :c:macro:`Py_TOLOWER`. .. data:: LC_COLLATE Locale category for sorting strings. The functions :func:`strcoll` and - :func:`strxfrm` of the :mod:`locale` module are affected. + :func:`strxfrm` of the :mod:`!locale` module are affected. .. data:: LC_TIME @@ -544,7 +566,7 @@ The :mod:`locale` module defines the following exception and functions: .. data:: LC_NUMERIC Locale category for formatting numbers. The functions :func:`format_string`, - :func:`atoi`, :func:`atof` and :func:`.str` of the :mod:`locale` module are + :func:`atoi`, :func:`atof` and :func:`.str` of the :mod:`!locale` module are affected by that category. All other numeric formatting operations are not affected. @@ -564,18 +586,6 @@ The :mod:`locale` module defines the following exception and functions: :func:`localeconv`. -Example:: - - >>> import locale - >>> loc = locale.getlocale() # get current locale - # use German locale; name might vary with platform - >>> locale.setlocale(locale.LC_ALL, 'de_DE') - >>> locale.strcoll('f\xe4n', 'foo') # compare a string containing an umlaut - >>> locale.setlocale(locale.LC_ALL, '') # use user's preferred locale - >>> locale.setlocale(locale.LC_ALL, 'C') # use default (C) locale - >>> locale.setlocale(locale.LC_ALL, loc) # restore saved locale - - Background, details, hints, tips and caveats -------------------------------------------- @@ -615,6 +625,61 @@ whose high bit is set (i.e., non-ASCII bytes) are never converted or considered part of a character class such as letter or whitespace. +.. _locale_name: + +Locale names +------------ + +The format of the locale name is platform dependent, and the set of supported +locales can depend on the system configuration. + +On Posix platforms, it usually has the format [1]_: + +.. productionlist:: locale_name + : language ["_" territory] ["." charset] ["@" modifier] + +where *language* is a two- or three-letter language code from `ISO 639`_, +*territory* is a two-letter country or region code from `ISO 3166`_, +*charset* is a locale encoding, and *modifier* is a script name, +a language subtag, a sort order identifier, or other locale modifier +(for example, "latin", "valencia", "stroke" and "euro"). + +On Windows, several formats are supported. [2]_ [3]_ +A subset of `IETF BCP 47`_ tags: + +.. productionlist:: locale_name + : language ["-" script] ["-" territory] ["." charset] + : language ["-" script] "-" territory "-" modifier + +where *language* and *territory* have the same meaning as in Posix, +*script* is a four-letter script code from `ISO 15924`_, +and *modifier* is a language subtag, a sort order identifier +or custom modifier (for example, "valencia", "stroke" or "x-python"). +Both hyphen (``'-'``) and underscore (``'_'``) separators are supported. +Only UTF-8 encoding is allowed for BCP 47 tags. + +Windows also supports locale names in the format: + +.. productionlist:: locale_name + : language ["_" territory] ["." charset] + +where *language* and *territory* are full names, such as "English" and +"United States", and *charset* is either a code page number (for example, "1252") +or UTF-8. +Only the underscore separator is supported in this format. + +The "C" locale is supported on all platforms. + +.. _ISO 639: https://www.iso.org/iso-639-language-code +.. _ISO 3166: https://www.iso.org/iso-3166-country-codes.html +.. _IETF BCP 47: https://www.rfc-editor.org/info/bcp47 +.. _ISO 15924: https://www.unicode.org/iso15924/ + +.. [1] `IEEE Std 1003.1-2024; 8.2 Internationalization Variables `_ +.. [2] `UCRT Locale names, Languages, and Country/Region strings `_ +.. [3] `Locale Names `_ + + .. _embedding-locale: For extension writers and programs that embed Python @@ -625,7 +690,7 @@ the current locale is. But since the return value can only be used portably to restore it, that is not very useful (except perhaps to find out whether or not the locale is ``C``). -When Python code uses the :mod:`locale` module to change the locale, this also +When Python code uses the :mod:`!locale` module to change the locale, this also affects the embedding application. If the embedding application doesn't want this to happen, it should remove the :mod:`!_locale` extension module (which does all the work) from the table of built-in modules in the :file:`config.c` file, diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 96cca3073fec7e7..3842c1e9d477d29 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -4,9 +4,6 @@ .. module:: logging.config :synopsis: Configuration of the logging module. -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - **Source code:** :source:`Lib/logging/config.py` .. sidebar:: Important @@ -28,7 +25,7 @@ Configuration functions ^^^^^^^^^^^^^^^^^^^^^^^ The following functions configure the logging module. They are located in the -:mod:`logging.config` module. Their use is optional --- you can configure the +:mod:`!logging.config` module. Their use is optional --- you can configure the logging module using these functions or by making calls to the main API (defined in :mod:`logging` itself) and defining handlers which are declared either in :mod:`logging` or :mod:`logging.handlers`. @@ -55,7 +52,7 @@ in :mod:`logging` itself) and defining handlers which are declared either in Parsing is performed by the :class:`DictConfigurator` class, whose constructor is passed the dictionary used for configuration, and - has a :meth:`configure` method. The :mod:`logging.config` module + has a :meth:`configure` method. The :mod:`!logging.config` module has a callable attribute :attr:`dictConfigClass` which is initially set to :class:`DictConfigurator`. You can replace the value of :attr:`dictConfigClass` with a @@ -127,12 +124,12 @@ in :mod:`logging` itself) and defining handlers which are declared either in application (e.g. based on command-line parameters or other aspects of the runtime environment) before being passed to ``fileConfig``. - .. versionchanged:: 3.10 - Added the *encoding* parameter. + .. versionchanged:: 3.10 + Added the *encoding* parameter. - .. versionchanged:: 3.12 - An exception will be thrown if the provided file - doesn't exist or is invalid or empty. + .. versionchanged:: 3.12 + An exception will be thrown if the provided file + doesn't exist or is invalid or empty. .. function:: listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None) diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index d74ef73ee284979..5152c7561fa1f26 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -4,9 +4,6 @@ .. module:: logging.handlers :synopsis: Handlers for the logging module. -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - **Source code:** :source:`Lib/logging/handlers.py` .. sidebar:: Important @@ -160,7 +157,7 @@ WatchedFileHandler .. currentmodule:: logging.handlers -The :class:`WatchedFileHandler` class, located in the :mod:`logging.handlers` +The :class:`WatchedFileHandler` class, located in the :mod:`!logging.handlers` module, is a :class:`FileHandler` which watches the file it is logging to. If the file changes, it is closed and reopened using the file name. @@ -213,7 +210,7 @@ for this value. BaseRotatingHandler ^^^^^^^^^^^^^^^^^^^ -The :class:`BaseRotatingHandler` class, located in the :mod:`logging.handlers` +The :class:`BaseRotatingHandler` class, located in the :mod:`!logging.handlers` module, is the base class for the rotating file handlers, :class:`RotatingFileHandler` and :class:`TimedRotatingFileHandler`. You should not need to instantiate this class, but it has attributes and methods you may @@ -307,7 +304,7 @@ For an example, see :ref:`cookbook-rotator-namer`. RotatingFileHandler ^^^^^^^^^^^^^^^^^^^ -The :class:`RotatingFileHandler` class, located in the :mod:`logging.handlers` +The :class:`RotatingFileHandler` class, located in the :mod:`!logging.handlers` module, supports rotation of disk log files. @@ -362,7 +359,7 @@ TimedRotatingFileHandler ^^^^^^^^^^^^^^^^^^^^^^^^ The :class:`TimedRotatingFileHandler` class, located in the -:mod:`logging.handlers` module, supports rotation of disk log files at certain +:mod:`!logging.handlers` module, supports rotation of disk log files at certain timed intervals. @@ -405,7 +402,8 @@ timed intervals. rollover interval. When computing the next rollover time for the first time (when the handler - is created), the last modification time of an existing log file, or else + is created), the creation time (if supported by the OS and file system) + or the last modification of an existing log file, or else the current time, is used to compute when the next rotation will occur. If the *utc* argument is true, times in UTC will be used; otherwise @@ -452,6 +450,10 @@ timed intervals. .. versionchanged:: 3.9 The *errors* parameter was added. + .. versionchanged:: next + Use the creation time instead of the last modification time, if supported by the OS and file system. + + .. method:: doRollover() Does a rollover, as described above. @@ -463,6 +465,7 @@ timed intervals. .. method:: getFilesToDelete() Returns a list of filenames which should be deleted as part of rollover. These + are the absolute paths of the oldest backup log files written by the handler. .. method:: shouldRollover(record) @@ -474,7 +477,7 @@ timed intervals. SocketHandler ^^^^^^^^^^^^^ -The :class:`SocketHandler` class, located in the :mod:`logging.handlers` module, +The :class:`SocketHandler` class, located in the :mod:`!logging.handlers` module, sends logging output to a network socket. The base class uses a TCP socket. @@ -570,7 +573,7 @@ sends logging output to a network socket. The base class uses a TCP socket. DatagramHandler ^^^^^^^^^^^^^^^ -The :class:`DatagramHandler` class, located in the :mod:`logging.handlers` +The :class:`DatagramHandler` class, located in the :mod:`!logging.handlers` module, inherits from :class:`SocketHandler` to support sending logging messages over UDP sockets. @@ -617,7 +620,7 @@ over UDP sockets. SysLogHandler ^^^^^^^^^^^^^ -The :class:`SysLogHandler` class, located in the :mod:`logging.handlers` module, +The :class:`SysLogHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a remote or local Unix syslog. @@ -796,7 +799,7 @@ supports sending logging messages to a remote or local Unix syslog. NTEventLogHandler ^^^^^^^^^^^^^^^^^ -The :class:`NTEventLogHandler` class, located in the :mod:`logging.handlers` +The :class:`NTEventLogHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a local Windows NT, Windows 2000 or Windows XP event log. Before you can use it, you need Mark Hammond's Win32 extensions for Python installed. @@ -863,7 +866,7 @@ extensions for Python installed. SMTPHandler ^^^^^^^^^^^ -The :class:`SMTPHandler` class, located in the :mod:`logging.handlers` module, +The :class:`SMTPHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to an email address via SMTP. @@ -904,7 +907,7 @@ supports sending logging messages to an email address via SMTP. MemoryHandler ^^^^^^^^^^^^^ -The :class:`MemoryHandler` class, located in the :mod:`logging.handlers` module, +The :class:`MemoryHandler` class, located in the :mod:`!logging.handlers` module, supports buffering of logging records in memory, periodically flushing them to a :dfn:`target` handler. Flushing occurs whenever the buffer is full, or when an event of a certain severity or greater is seen. @@ -984,7 +987,7 @@ should, then :meth:`flush` is expected to do the flushing. HTTPHandler ^^^^^^^^^^^ -The :class:`HTTPHandler` class, located in the :mod:`logging.handlers` module, +The :class:`HTTPHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a web server, using either ``GET`` or ``POST`` semantics. @@ -1036,7 +1039,7 @@ QueueHandler .. versionadded:: 3.2 -The :class:`QueueHandler` class, located in the :mod:`logging.handlers` module, +The :class:`QueueHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a queue, such as those implemented in the :mod:`queue` or :mod:`multiprocessing` modules. @@ -1129,7 +1132,7 @@ QueueListener .. versionadded:: 3.2 -The :class:`QueueListener` class, located in the :mod:`logging.handlers` +The :class:`QueueListener` class, located in the :mod:`!logging.handlers` module, supports receiving logging messages from a queue, such as those implemented in the :mod:`queue` or :mod:`multiprocessing` modules. The messages are received from a queue in an internal thread and passed, on diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index e37683e32d01f42..a3d117c10702414 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -4,9 +4,6 @@ .. module:: logging :synopsis: Flexible event logging system for applications. -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - **Source code:** :source:`Lib/logging/__init__.py` .. index:: pair: Errors; logging @@ -671,8 +668,7 @@ Formatter Objects which is just the logged message. :type fmt: str - :param datefmt: A format string in the given *style* for - the date/time portion of the logged output. + :param datefmt: A format string for the date/time portion of the logged output. If not specified, the default described in :meth:`formatTime` is used. :type datefmt: str @@ -680,7 +676,7 @@ Formatter Objects how the format string will be merged with its data: using one of :ref:`old-string-formatting` (``%``), :meth:`str.format` (``{``) or :class:`string.Template` (``$``). This only applies to - *fmt* and *datefmt* (e.g. ``'%(message)s'`` versus ``'{message}'``), + *fmt* (e.g. ``'%(message)s'`` versus ``'{message}'``), not to the actual log messages passed to the logging methods. However, there are :ref:`other ways ` to use ``{``- and ``$``-formatting for log messages. @@ -694,7 +690,7 @@ Formatter Objects :param defaults: A dictionary with default values to use in custom fields. For example, ``logging.Formatter('%(ip)s %(message)s', defaults={"ip": None})`` - :type defaults: dict[str, Any] + :type defaults: dict[str, typing.Any] .. versionchanged:: 3.2 Added the *style* parameter. @@ -1012,6 +1008,11 @@ the options available to you. | exc_info | You shouldn't need to | Exception tuple (à la ``sys.exc_info``) or, | | | format this yourself. | if no exception has occurred, ``None``. | +----------------+-------------------------+-----------------------------------------------+ +| exc_text | You shouldn't need to | Exception information formatted as a string. | +| | format this yourself. | This is set when :meth:`Formatter.format` is | +| | | invoked, or ``None`` if no exception has | +| | | occurred. | ++----------------+-------------------------+-----------------------------------------------+ | filename | ``%(filename)s`` | Filename portion of ``pathname``. | +----------------+-------------------------+-----------------------------------------------+ | funcName | ``%(funcName)s`` | Name of function containing the logging call. | @@ -1083,12 +1084,13 @@ LoggerAdapter Objects information into logging calls. For a usage example, see the section on :ref:`adding contextual information to your logging output `. -.. class:: LoggerAdapter(logger, extra, merge_extra=False) +.. class:: LoggerAdapter(logger, extra=None, merge_extra=False) Returns an instance of :class:`LoggerAdapter` initialized with an - underlying :class:`Logger` instance, a dict-like object (*extra*), and a - boolean (*merge_extra*) indicating whether or not the *extra* argument of - individual log calls should be merged with the :class:`LoggerAdapter` extra. + underlying :class:`Logger` instance, an optional dict-like object (*extra*), + and an optional boolean (*merge_extra*) indicating whether or not + the *extra* argument of individual log calls should be merged with + the :class:`LoggerAdapter` extra. The default behavior is to ignore the *extra* argument of individual log calls and only use the one of the :class:`LoggerAdapter` instance @@ -1128,16 +1130,20 @@ information into logging calls. For a usage example, see the section on Attribute :attr:`!manager` and method :meth:`!_log` were added, which delegate to the underlying logger and allow adapters to be nested. + .. versionchanged:: 3.10 + + The *extra* argument is now optional. + .. versionchanged:: 3.13 - The *merge_extra* argument was added. + The *merge_extra* parameter was added. Thread Safety ------------- The logging module is intended to be thread-safe without any special work -needing to be done by its clients. It achieves this though using threading +needing to be done by its clients. It achieves this through using threading locks; there is one lock to serialize access to the module's shared data, and each handler also creates a lock to serialize access to its underlying I/O. @@ -1542,7 +1548,7 @@ Module-Level Attributes Integration with the warnings module ------------------------------------ -The :func:`captureWarnings` function can be used to integrate :mod:`logging` +The :func:`captureWarnings` function can be used to integrate :mod:`!logging` with the :mod:`warnings` module. .. function:: captureWarnings(capture) @@ -1573,7 +1579,7 @@ with the :mod:`warnings` module. library. `Original Python logging package `_ - This is the original source for the :mod:`logging` package. The version of the + This is the original source for the :mod:`!logging` package. The version of the package available from this site is suitable for use with Python 1.5.2, 2.1.x - and 2.2.x, which do not include the :mod:`logging` package in the standard + and 2.2.x, which do not include the :mod:`!logging` package in the standard library. diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 69f7cb8d48d7aee..cd72174d54f6e62 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -4,9 +4,6 @@ .. module:: lzma :synopsis: A Python wrapper for the liblzma compression library. -.. moduleauthor:: Nadeem Vawda -.. sectionauthor:: Nadeem Vawda - .. versionadded:: 3.3 **Source code:** :source:`Lib/lzma.py` @@ -23,6 +20,8 @@ module. Note that :class:`LZMAFile` and :class:`bz2.BZ2File` are *not* thread-safe, so if you need to use a single :class:`LZMAFile` instance from multiple threads, it is necessary to protect it with a lock. +.. include:: ../includes/optional-module.rst + .. exception:: LZMAError @@ -357,12 +356,26 @@ options. Valid filter IDs are as follows: * Branch-Call-Jump (BCJ) filters: - * :const:`FILTER_X86` - * :const:`FILTER_IA64` - * :const:`FILTER_ARM` - * :const:`FILTER_ARMTHUMB` - * :const:`FILTER_POWERPC` - * :const:`FILTER_SPARC` + * :const:`!FILTER_X86` + * :const:`!FILTER_IA64` + * :const:`!FILTER_ARM` + * :const:`!FILTER_ARMTHUMB` + * :const:`!FILTER_POWERPC` + * :const:`!FILTER_SPARC` + + The above work on all lzma runtime library versions. + + * :const:`!FILTER_ARM64` + + Only works if the lzma version is 5.4.0 or later. + + .. versionadded:: next + + * :const:`!FILTER_RISCV` + + Only works if the lzma version is 5.6.0 or later. + + .. versionadded:: next A filter chain can consist of up to 4 filters, and cannot be empty. The last filter in the chain must be a compression filter, and any other filters must be diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index e8a96f29ea185e6..5b9741bdbcad19e 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -4,9 +4,6 @@ .. module:: mailbox :synopsis: Manipulate mailboxes in various formats -.. moduleauthor:: Gregory K. Johnson -.. sectionauthor:: Gregory K. Johnson - **Source code:** :source:`Lib/mailbox.py` -------------- @@ -78,6 +75,14 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. message. Failing to lock the mailbox runs the risk of losing messages or corrupting the entire mailbox. + The :class:`!Mailbox` class supports the :keyword:`with` statement. When used + as a context manager, :class:`!Mailbox` calls :meth:`lock` when the context is entered, + returns the mailbox object as the context object, and at context end calls :meth:`close`, + thereby releasing the lock. + + .. versionchanged:: 3.15 + Support for the :keyword:`with` statement was added. + :class:`!Mailbox` instances have the following methods: @@ -917,7 +922,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. copied; furthermore, any format-specific information is converted insofar as possible if *message* is a :class:`!Message` instance. If *message* is a string, a byte string, - or a file, it should contain an :rfc:`2822`\ -compliant message, which is read + or a file, it should contain an :rfc:`5322`\ -compliant message, which is read and parsed. Files should be open in binary mode, but text mode files are accepted for backward compatibility. @@ -1025,7 +1030,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: remove_flag(flag) Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than + remove more than one flag at a time, *flag* may be a string of more than one character. If "info" contains experimental information rather than flags, the current "info" is not modified. @@ -1190,7 +1195,7 @@ When a :class:`!MaildirMessage` instance is created based upon a .. method:: remove_flag(flag) Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than + remove more than one flag at a time, *flag* may be a string of more than one character. When an :class:`!mboxMessage` instance is created based upon a @@ -1562,7 +1567,7 @@ When a :class:`!BabylMessage` instance is created based upon an .. method:: remove_flag(flag) Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than + remove more than one flag at a time, *flag* may be a string of more than one character. When an :class:`!MMDFMessage` instance is created based upon a @@ -1641,7 +1646,7 @@ The following exception classes are defined in the :mod:`!mailbox` module: .. exception:: Error() - The based class for all other module-specific exceptions. + The base class for all other module-specific exceptions. .. exception:: NoSuchMailboxError() @@ -1661,7 +1666,7 @@ The following exception classes are defined in the :mod:`!mailbox` module: Raised when some mailbox-related condition beyond the control of the program causes it to be unable to proceed, such as when failing to acquire a lock that - another program already holds a lock, or when a uniquely generated file name + another program already holds, or when a uniquely generated file name already exists. diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index e8e9071a5c9ef49..4fe34f0a3a3f911 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -20,7 +20,7 @@ rarely does). [#]_ This is not a general "persistence" module. For general persistence and transfer of Python objects through RPC calls, see the modules :mod:`pickle` and -:mod:`shelve`. The :mod:`marshal` module exists mainly to support reading and +:mod:`shelve`. The :mod:`!marshal` module exists mainly to support reading and writing the "pseudo-compiled" code for Python modules of :file:`.pyc` files. Therefore, the Python maintainers reserve the right to modify the marshal format in backward incompatible ways should the need arise. @@ -34,7 +34,7 @@ supports a substantially wider range of objects than marshal. .. warning:: - The :mod:`marshal` module is not intended to be secure against erroneous or + The :mod:`!marshal` module is not intended to be secure against erroneous or maliciously constructed data. Never unmarshal data received from an untrusted or unauthenticated source. @@ -51,8 +51,9 @@ this module. The following types are supported: * Strings (:class:`str`) and :class:`bytes`. :term:`Bytes-like objects ` like :class:`bytearray` are marshalled as :class:`!bytes`. -* Containers: :class:`tuple`, :class:`list`, :class:`set`, :class:`frozenset`, - and (since :data:`version` 5), :class:`slice`. +* Containers: :class:`tuple`, :class:`list`, :class:`dict`, :class:`frozendict` + (since :data:`version` 6), :class:`set`, :class:`frozenset`, and + :class:`slice` (since :data:`version` 5). It should be understood that these are supported only if the values contained therein are themselves supported. Recursive containers are supported since :data:`version` 3. @@ -71,6 +72,10 @@ this module. The following types are supported: Added format version 5, which allows marshalling slices. +.. versionchanged:: 3.15 + + Added format version 6, which allows marshalling :class:`frozendict`. + The module defines these functions: @@ -173,6 +178,8 @@ In addition, the following constants are defined: 4 Python 3.4 Efficient representation of short strings ------- --------------- ---------------------------------------------------- 5 Python 3.14 Support for :class:`slice` objects + ------- --------------- ---------------------------------------------------- + 6 Python 3.15 Support for :class:`frozendict` objects ======= =============== ==================================================== diff --git a/Doc/library/math.integer.rst b/Doc/library/math.integer.rst new file mode 100644 index 000000000000000..c3f34cdfd85410c --- /dev/null +++ b/Doc/library/math.integer.rst @@ -0,0 +1,85 @@ +:mod:`!math.integer` --- integer-specific mathematics functions +=============================================================== + +.. module:: math.integer + :synopsis: Integer-specific mathematics functions. + +.. versionadded:: 3.15 + +-------------- + +This module provides access to the mathematical functions defined for integer arguments. +These functions accept integers and objects that implement the +:meth:`~object.__index__` method which is used to convert the object to an integer +number. + +The following functions are provided by this module. All return values are +computed exactly and are integers. + + +.. function:: comb(n, k, /) + + Return the number of ways to choose *k* items from *n* items without repetition + and without order. + + Evaluates to ``n! / (k! * (n - k)!)`` when ``k <= n`` and evaluates + to zero when ``k > n``. + + Also called the binomial coefficient because it is equivalent + to the coefficient of k-th term in polynomial expansion of + ``(1 + x)ⁿ``. + + Raises :exc:`ValueError` if either of the arguments are negative. + + +.. function:: factorial(n, /) + + Return factorial of the nonnegative integer *n*. + + +.. function:: gcd(*integers) + + Return the greatest common divisor of the specified integer arguments. + If any of the arguments is nonzero, then the returned value is the largest + positive integer that is a divisor of all arguments. If all arguments + are zero, then the returned value is ``0``. ``gcd()`` without arguments + returns ``0``. + + +.. function:: isqrt(n, /) + + Return the integer square root of the nonnegative integer *n*. This is the + floor of the exact square root of *n*, or equivalently the greatest integer + *a* such that *a*\ ² |nbsp| ≤ |nbsp| *n*. + + For some applications, it may be more convenient to have the least integer + *a* such that *n* |nbsp| ≤ |nbsp| *a*\ ², or in other words the ceiling of + the exact square root of *n*. For positive *n*, this can be computed using + ``a = 1 + isqrt(n - 1)``. + + + .. |nbsp| unicode:: 0xA0 + :trim: + + +.. function:: lcm(*integers) + + Return the least common multiple of the specified integer arguments. + If all arguments are nonzero, then the returned value is the smallest + positive integer that is a multiple of all arguments. If any of the arguments + is zero, then the returned value is ``0``. ``lcm()`` without arguments + returns ``1``. + + +.. function:: perm(n, k=None, /) + + Return the number of ways to choose *k* items from *n* items + without repetition and with order. + + Evaluates to ``n! / (n - k)!`` when ``k <= n`` and evaluates + to zero when ``k > n``. + + If *k* is not specified or is ``None``, then *k* defaults to *n* + and the function returns ``n!``. + + Raises :exc:`ValueError` if either of the arguments are negative. diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 55f2de07553d563..efe411e5a43f27d 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -27,15 +27,6 @@ noted otherwise, all return values are floats. ==================================================== ============================================ -**Number-theoretic functions** --------------------------------------------------------------------------------------------------- -:func:`comb(n, k) ` Number of ways to choose *k* items from *n* items without repetition and without order -:func:`factorial(n) ` *n* factorial -:func:`gcd(*integers) ` Greatest common divisor of the integer arguments -:func:`isqrt(n) ` Integer square root of a nonnegative integer *n* -:func:`lcm(*integers) ` Least common multiple of the integer arguments -:func:`perm(n, k) ` Number of ways to choose *k* items from *n* items without repetition and with order - **Floating point arithmetic** -------------------------------------------------------------------------------------------------- :func:`ceil(x) ` Ceiling of *x*, the smallest integer greater than or equal to *x* @@ -92,13 +83,20 @@ noted otherwise, all return values are floats. **Trigonometric functions** -------------------------------------------------------------------------------------------------- -:func:`acos(x) ` Arc cosine of *x* -:func:`asin(x) ` Arc sine of *x* -:func:`atan(x) ` Arc tangent of *x* -:func:`atan2(y, x) ` ``atan(y / x)`` -:func:`cos(x) ` Cosine of *x* -:func:`sin(x) ` Sine of *x* -:func:`tan(x) ` Tangent of *x* +:func:`acos(x) ` Arc cosine of *x*, in radians +:func:`acospi(x) ` Arc cosine of *x*, in half-turns +:func:`asin(x) ` Arc sine of *x*, in radians +:func:`asinpi(x) ` Arc sine of *x*, in half-turns +:func:`atan(x) ` Arc tangent of *x*, in radians +:func:`atanpi(x) ` Arc tangent of *x*, in half-turns +:func:`atan2(y, x) ` ``atan(y / x)``, in radians +:func:`atan2pi(y, x) ` ``atan(y / x)``, in half-turns +:func:`cos(x) ` Cosine of *x* radians +:func:`cospi(x) ` Cosine of *x⋅π* radians +:func:`sin(x) ` Sine of *x* radians +:func:`sinpi(x) ` Sine of *x⋅π* radians +:func:`tan(x) ` Tangent of *x* radians +:func:`tanpi(x) ` Tangent of *x⋅π* radians **Hyperbolic functions** -------------------------------------------------------------------------------------------------- @@ -126,92 +124,6 @@ noted otherwise, all return values are floats. ==================================================== ============================================ -Number-theoretic functions --------------------------- - -.. function:: comb(n, k) - - Return the number of ways to choose *k* items from *n* items without repetition - and without order. - - Evaluates to ``n! / (k! * (n - k)!)`` when ``k <= n`` and evaluates - to zero when ``k > n``. - - Also called the binomial coefficient because it is equivalent - to the coefficient of k-th term in polynomial expansion of - ``(1 + x)ⁿ``. - - Raises :exc:`TypeError` if either of the arguments are not integers. - Raises :exc:`ValueError` if either of the arguments are negative. - - .. versionadded:: 3.8 - - -.. function:: factorial(n) - - Return factorial of the nonnegative integer *n*. - - .. versionchanged:: 3.10 - Floats with integral values (like ``5.0``) are no longer accepted. - - -.. function:: gcd(*integers) - - Return the greatest common divisor of the specified integer arguments. - If any of the arguments is nonzero, then the returned value is the largest - positive integer that is a divisor of all arguments. If all arguments - are zero, then the returned value is ``0``. ``gcd()`` without arguments - returns ``0``. - - .. versionadded:: 3.5 - - .. versionchanged:: 3.9 - Added support for an arbitrary number of arguments. Formerly, only two - arguments were supported. - - -.. function:: isqrt(n) - - Return the integer square root of the nonnegative integer *n*. This is the - floor of the exact square root of *n*, or equivalently the greatest integer - *a* such that *a*\ ² |nbsp| ≤ |nbsp| *n*. - - For some applications, it may be more convenient to have the least integer - *a* such that *n* |nbsp| ≤ |nbsp| *a*\ ², or in other words the ceiling of - the exact square root of *n*. For positive *n*, this can be computed using - ``a = 1 + isqrt(n - 1)``. - - .. versionadded:: 3.8 - - -.. function:: lcm(*integers) - - Return the least common multiple of the specified integer arguments. - If all arguments are nonzero, then the returned value is the smallest - positive integer that is a multiple of all arguments. If any of the arguments - is zero, then the returned value is ``0``. ``lcm()`` without arguments - returns ``1``. - - .. versionadded:: 3.9 - - -.. function:: perm(n, k=None) - - Return the number of ways to choose *k* items from *n* items - without repetition and with order. - - Evaluates to ``n! / (n - k)!`` when ``k <= n`` and evaluates - to zero when ``k > n``. - - If *k* is not specified or is ``None``, then *k* defaults to *n* - and the function returns ``n!``. - - Raises :exc:`TypeError` if either of the arguments are not integers. - Raises :exc:`ValueError` if either of the arguments are negative. - - .. versionadded:: 3.8 - - Floating point arithmetic ------------------------- @@ -259,7 +171,7 @@ Floating point arithmetic is, :func:`!fmax` is not required to be sensitive to the sign of such operands (see Annex F of the C11 standard, §F.10.0.3 and §F.10.9.2). - .. versionadded:: next + .. versionadded:: 3.15 .. function:: fmin(x, y) @@ -271,7 +183,7 @@ Floating point arithmetic is, :func:`!fmin` is not required to be sensitive to the sign of such operands (see Annex F of the C11 standard, §F.10.0.3 and §F.10.9.3). - .. versionadded:: next + .. versionadded:: 3.15 .. function:: fmod(x, y) @@ -350,10 +262,12 @@ Floating point manipulation functions .. function:: frexp(x) - Return the mantissa and exponent of *x* as the pair ``(m, e)``. *m* is a float - and *e* is an integer such that ``x == m * 2**e`` exactly. If *x* is zero, - returns ``(0.0, 0)``, otherwise ``0.5 <= abs(m) < 1``. This is used to "pick - apart" the internal representation of a float in a portable way. + Return the mantissa and exponent of *x* as the pair ``(m, e)``. + If *x* is a finite nonzero number, then *m* is a float with + ``0.5 <= abs(m) < 1.0`` and an integer *e* is such that + ``x == m * 2**e`` exactly. Else, return ``(x, 0)``. + This is used to "pick apart" the internal representation of + a float in a portable way. Note that :func:`frexp` has a different call/return pattern than its C equivalents: it takes a single argument and return a pair of @@ -408,7 +322,7 @@ Floating point manipulation functions nonzero number that is not a subnormal (see :func:`issubnormal`). Return ``False`` otherwise. - .. versionadded:: next + .. versionadded:: 3.15 .. function:: issubnormal(x) @@ -417,7 +331,7 @@ Floating point manipulation functions nonzero number with a magnitude smaller than :data:`sys.float_info.min`. Return ``False`` otherwise. - .. versionadded:: next + .. versionadded:: 3.15 .. function:: isinf(x) @@ -464,7 +378,7 @@ Floating point manipulation functions This is useful to detect the sign bit of zeroes, infinities and NaNs. - .. versionadded:: next + .. versionadded:: 3.15 .. function:: ulp(x) @@ -601,7 +515,7 @@ Summation and product functions Roughly equivalent to:: - sqrt(sum((px - qx) ** 2.0 for px, qx in zip(p, q))) + sqrt(sum((px - qx) ** 2.0 for px, qx in zip(p, q, strict=True))) .. versionadded:: 3.8 @@ -692,18 +606,42 @@ Trigonometric functions ``pi``. +.. function:: acospi(x) + + Return the arc cosine of *x*, in half-turns. The result is between ``0`` and + ``1``. + + .. versionadded:: next + + .. function:: asin(x) Return the arc sine of *x*, in radians. The result is between ``-pi/2`` and ``pi/2``. +.. function:: asinpi(x) + + Return the arc sine of *x*, in half-turns. The result is between ``-0.5`` and + ``0.5``. + + .. versionadded:: next + + .. function:: atan(x) Return the arc tangent of *x*, in radians. The result is between ``-pi/2`` and ``pi/2``. +.. function:: atanpi(x) + + Return the arc tangent of *x*, in half-turns. The result is between ``-0.5`` and + ``0.5``. + + .. versionadded:: next + + .. function:: atan2(y, x) Return ``atan(y / x)``, in radians. The result is between ``-pi`` and ``pi``. @@ -714,21 +652,54 @@ Trigonometric functions -1)`` is ``-3*pi/4``. +.. function:: atan2pi(y, x) + + Return ``atanpi(y / x)``, in half-turns. The result is between ``-1`` and ``1``. + The vector in the plane from the origin to point ``(x, y)`` makes this angle + with the positive X axis. The point of :func:`atan2pi` is that the signs of both + inputs are known to it, so it can compute the correct quadrant for the angle. + For example, ``atanpi(1)`` and ``atan2pi(1, 1)`` are both ``0.25``, but + ``atan2pi(-1, -1)`` is ``-0.75``. + + .. versionadded:: next + + .. function:: cos(x) Return the cosine of *x* radians. +.. function:: cospi(x) + + Return the cosine of *x* half-turns (*x⋅π* radians). + + .. versionadded:: next + + .. function:: sin(x) Return the sine of *x* radians. +.. function:: sinpi(x) + + Return the sine of *x* half-turns (*x⋅π* radians). + + .. versionadded:: next + + .. function:: tan(x) Return the tangent of *x* radians. +.. function:: tanpi(x) + + Return the tangent of *x* half-turns (*x⋅π* radians). + + .. versionadded:: next + + Hyperbolic functions -------------------- @@ -812,6 +783,74 @@ Special functions .. versionadded:: 3.2 +Number-theoretic functions +-------------------------- + +For backward compatibility, the :mod:`!math` module provides also aliases of +the following functions from the :mod:`math.integer` module: + +.. list-table:: + + * - .. function:: comb(n, k) + :no-typesetting: + + :func:`comb(n, k) ` + - Number of ways to choose *k* items from *n* items without repetition + and without order + + * - .. function:: factorial(n) + :no-typesetting: + + :func:`factorial(n) ` + - *n* factorial + + * - .. function:: gcd(*integers) + :no-typesetting: + + :func:`gcd(*integers) ` + - Greatest common divisor of the integer arguments + + * - .. function:: isqrt(n) + :no-typesetting: + + :func:`isqrt(n) ` + - Integer square root of a nonnegative integer *n* + + * - .. function:: lcm(*integers) + :no-typesetting: + + :func:`lcm(*integers) ` + - Least common multiple of the integer arguments + + * - .. function:: perm(n, k) + :no-typesetting: + + :func:`perm(n, k) ` + - Number of ways to choose *k* items from *n* items without repetition + and with order + +.. versionadded:: 3.5 + The :func:`gcd` function. + +.. versionadded:: 3.8 + The :func:`comb`, :func:`perm` and :func:`isqrt` functions. + +.. versionadded:: 3.9 + The :func:`lcm` function. + +.. versionchanged:: 3.9 + Added support for an arbitrary number of arguments in the :func:`gcd` + function. + Formerly, only two arguments were supported. + +.. versionchanged:: 3.10 + Floats with integral values (like ``5.0``) are no longer accepted in the + :func:`factorial` function. + +.. soft-deprecated:: 3.15 + Use the :mod:`math.integer` functions instead of these aliases. + + Constants --------- @@ -872,7 +911,7 @@ Constants .. impl-detail:: - The :mod:`math` module consists mostly of thin wrappers around the platform C + The :mod:`!math` module consists mostly of thin wrappers around the platform C math library functions. Behavior in exceptional cases follows Annex F of the C99 standard where appropriate. The current implementation will raise :exc:`ValueError` for invalid operations like ``sqrt(-1.0)`` or ``log(0.0)`` @@ -894,5 +933,5 @@ Constants Module :mod:`cmath` Complex number versions of many of these functions. -.. |nbsp| unicode:: 0xA0 - :trim: + Module :mod:`math.integer` + Integer-specific mathematics functions. diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 13511b16a0ed8ce..5c29fff146eef00 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -4,15 +4,13 @@ .. module:: mimetypes :synopsis: Mapping of filename extensions to MIME types. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/mimetypes.py` .. index:: pair: MIME; content type -------------- -The :mod:`mimetypes` module converts between a filename or URL and the MIME type +The :mod:`!mimetypes` module converts between a filename or URL and the MIME type associated with the filename extension. Conversions are provided from filename to MIME type and from MIME type to filename extension; encodings are not supported for the latter conversion. @@ -41,8 +39,8 @@ the information :func:`init` sets up. (e.g. :program:`compress` or :program:`gzip`). The encoding is suitable for use as a :mailheader:`Content-Encoding` header, **not** as a :mailheader:`Content-Transfer-Encoding` header. The mappings are table driven. - Encoding suffixes are case sensitive; type suffixes are first tried case - sensitively, then case insensitively. + Encoding suffixes are case-sensitive. Suffix mappings and type suffixes are + first tried case-sensitively, then case-insensitively. The optional *strict* argument is a flag specifying whether the list of known MIME types is limited to only the official types `registered with IANA @@ -56,8 +54,8 @@ the information :func:`init` sets up. .. versionchanged:: 3.8 Added support for *url* being a :term:`path-like object`. - .. deprecated:: 3.13 - Passing a file path instead of URL is :term:`soft deprecated`. + .. soft-deprecated:: 3.13 + Passing a file path instead of URL. Use :func:`guess_file_type` for this. @@ -131,10 +129,20 @@ behavior of the module. Add a mapping from the MIME type *type* to the extension *ext*. When the extension is already known, the new type will replace the old one. When the type is already known the extension will be added to the list of known extensions. + Valid extensions are empty or start with a ``'.'``. + + Registered lower-case extensions are matched case-insensitively. When *strict* is ``True`` (the default), the mapping will be added to the official MIME types, otherwise to the non-standard ones. + .. deprecated:: 3.14 + *ext* values that do not start with ``'.'`` are deprecated. + + .. versionchanged:: next + *ext* now must start with ``'.'``. Otherwise :exc:`ValueError` is raised. + + .. data:: inited @@ -196,7 +204,7 @@ MimeTypes objects The :class:`MimeTypes` class may be useful for applications which may want more than one MIME-type database; it provides an interface similar to the one of the -:mod:`mimetypes` module. +:mod:`!mimetypes` module. .. class:: MimeTypes(filenames=(), strict=True) @@ -306,12 +314,16 @@ than one MIME-type database; it provides an interface similar to the one of the extension is already known, the new type will replace the old one. When the type is already known the extension will be added to the list of known extensions. + Registered lower-case extensions are matched case-insensitively. + When *strict* is ``True`` (the default), the mapping will be added to the official MIME types, otherwise to the non-standard ones. - .. deprecated-removed:: 3.14 3.16 - Invalid, undotted extensions will raise a - :exc:`ValueError` in Python 3.16. + .. deprecated:: 3.14 + *ext* values that do not start with ``'.'`` are deprecated. + + .. versionchanged:: next + *ext* now must start with ``'.'``. Otherwise :exc:`ValueError` is raised. .. _mimetypes-cli: @@ -350,7 +362,7 @@ it converts file extensions to MIME types. For each ``type`` entry, the script writes a line into the standard output stream. If an unknown type occurs, it writes an error message into the -standard error stream and exits with the return code ``1``. +standard output stream and exits with the return code ``1``. .. mimetypes-cli-example: @@ -377,7 +389,7 @@ interface: $ # get a MIME type for a rare file extension $ python -m mimetypes filename.pict - error: unknown extension of filename.pict + error: media type unknown for filename.pict $ # now look in the extended database built into Python $ python -m mimetypes --lenient filename.pict @@ -399,7 +411,8 @@ interface: $ python -m mimetypes filename.sh filename.nc filename.xxx filename.txt type: application/x-sh encoding: None type: application/x-netcdf encoding: None - error: unknown extension of filename.xxx + error: media type unknown for filename.xxx + type: text/plain encoding: None $ # try to feed an unknown MIME type $ python -m mimetypes --extension audio/aac audio/opus audio/future audio/x-wav diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 8fca79b23e4e156..774d05fdcaa3007 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -48,10 +48,11 @@ update the underlying file. To map anonymous memory, -1 should be passed as the fileno along with the length. -.. class:: mmap(fileno, length, tagname=None, access=ACCESS_DEFAULT, offset=0) +.. class:: mmap(fileno, length, tagname=None, \ + access=ACCESS_DEFAULT, offset=0, *, trackfd=True) **(Windows version)** Maps *length* bytes from the file specified by the - file handle *fileno*, and creates a mmap object. If *length* is larger + file descriptor *fileno*, and creates a mmap object. If *length* is larger than the current size of the file, the file is extended to contain *length* bytes. If *length* is ``0``, the maximum length of the map is the current size of the file, except that if the file is empty Windows raises an @@ -69,6 +70,17 @@ To map anonymous memory, -1 should be passed as the fileno along with the length will be relative to the offset from the beginning of the file. *offset* defaults to 0. *offset* must be a multiple of the :const:`ALLOCATIONGRANULARITY`. + If *trackfd* is ``False``, the file handle corresponding to *fileno* will + not be duplicated, and the resulting :class:`!mmap` object will not + be associated with the map's underlying file. + This means that the :meth:`~mmap.mmap.size` and :meth:`~mmap.mmap.resize` + methods will fail. + This mode is useful to limit the number of open file handles. + The original file can be renamed (but not deleted) after closing *fileno*. + + .. versionchanged:: 3.15 + The *trackfd* parameter was added. + .. audit-event:: mmap.__new__ fileno,length,access,offset mmap.mmap .. class:: mmap(fileno, length, flags=MAP_SHARED, prot=PROT_WRITE|PROT_READ, \ @@ -200,7 +212,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length Writable :term:`bytes-like object` is now accepted. - .. method:: flush([offset[, size]]) + .. method:: flush([offset[, size]], *, flags=MS_SYNC) Flushes changes made to the in-memory copy of a file back to disk. Without use of this call there is no guarantee that changes are written back before @@ -209,6 +221,12 @@ To map anonymous memory, -1 should be passed as the fileno along with the length whole extent of the mapping is flushed. *offset* must be a multiple of the :const:`PAGESIZE` or :const:`ALLOCATIONGRANULARITY`. + The *flags* parameter specifies the synchronization behavior. + *flags* must be one of the :ref:`MS_* constants ` available + on the system. + + On Windows, the *flags* parameter is ignored. + ``None`` is returned to indicate success. An exception is raised when the call failed. @@ -217,6 +235,15 @@ To map anonymous memory, -1 should be passed as the fileno along with the length on error under Windows. A zero value was returned on success; an exception was raised on error under Unix. + .. versionchanged:: 3.15 + Allow specifying *offset* without *size*. Previously, both *offset* + and *size* parameters were required together. Now *offset* can be + specified alone, and the flush operation will extend from *offset* + to the end of the mmap. + + .. versionchanged:: 3.15 + Added *flags* parameter to control synchronization behavior. + .. method:: madvise(option[, start[, length]]) @@ -277,6 +304,8 @@ To map anonymous memory, -1 should be passed as the fileno along with the length pagefile) will silently create a new map with the original data copied over up to the length of the new size. + Availability: Windows and systems with the ``mremap()`` system call. + .. versionchanged:: 3.11 Correctly fails if attempting to resize when another map is held Allows resize against an anonymous map on Windows @@ -308,10 +337,25 @@ To map anonymous memory, -1 should be passed as the fileno along with the length .. versionadded:: 3.13 + .. method:: set_name(name, /) + + Annotate the memory mapping with the given *name* for easier identification + in ``/proc//maps`` if the kernel supports the feature and :option:`-X dev <-X>` is passed + to Python or if Python is built in :ref:`debug mode `. + The length of *name* must not exceed 67 bytes including the ``'\0'`` terminator. + + .. availability:: Linux >= 5.17 (kernel built with ``CONFIG_ANON_VMA_NAME`` option) + + .. versionadded:: 3.15 + .. method:: size() Return the length of the file, which can be larger than the size of the memory-mapped area. + For an anonymous mapping, return its size. + + .. versionchanged:: 3.15 + Anonymous mappings are now supported on Unix. .. method:: tell() @@ -426,3 +470,22 @@ MAP_* Constants :data:`MAP_TPRO`, :data:`MAP_TRANSLATED_ALLOW_EXECUTE`, and :data:`MAP_UNIX03` constants. +.. _ms-constants: + +MS_* Constants +++++++++++++++ + +.. data:: MS_SYNC + MS_ASYNC + MS_INVALIDATE + + These flags control the synchronization behavior for :meth:`mmap.flush`: + + * :data:`MS_SYNC` - Synchronous flush: writes are scheduled and the call + blocks until they are physically written to storage. + * :data:`MS_ASYNC` - Asynchronous flush: writes are scheduled but the call + returns immediately without waiting for completion. + * :data:`MS_INVALIDATE` - Invalidate cached data: invalidates other mappings + of the same file so they can see the changes. + + .. versionadded:: 3.15 diff --git a/Doc/library/modulefinder.rst b/Doc/library/modulefinder.rst index 823d853f1ed8ebe..d26dcbd5f687257 100644 --- a/Doc/library/modulefinder.rst +++ b/Doc/library/modulefinder.rst @@ -4,8 +4,6 @@ .. module:: modulefinder :synopsis: Find modules used by a script. -.. sectionauthor:: A.M. Kuchling - **Source code:** :source:`Lib/modulefinder.py` -------------- diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index 327cc3602b1a773..6b49c1a9ccd6e11 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -2,10 +2,9 @@ =========================================================== .. module:: msvcrt - :platform: Windows :synopsis: Miscellaneous useful routines from the MS VC++ runtime. -.. sectionauthor:: Fred L. Drake, Jr. +**Source code:** :source:`PC/msvcrtmodule.c` -------------- @@ -22,6 +21,8 @@ api. The normal API deals only with ASCII characters and is of limited use for internationalized applications. The wide char API should be used where ever possible. +.. availability:: Windows. + .. versionchanged:: 3.3 Operations in this module now raise :exc:`OSError` where :exc:`IOError` was raised. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index c80f78e614818eb..2d13053915830b0 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -13,17 +13,16 @@ Introduction ------------ -:mod:`multiprocessing` is a package that supports spawning processes using an -API similar to the :mod:`threading` module. The :mod:`multiprocessing` package +:mod:`!multiprocessing` is a package that supports spawning processes using an +API similar to the :mod:`threading` module. The :mod:`!multiprocessing` package offers both local and remote concurrency, effectively side-stepping the :term:`Global Interpreter Lock ` by using subprocesses instead of threads. Due -to this, the :mod:`multiprocessing` module allows the programmer to fully +to this, the :mod:`!multiprocessing` module allows the programmer to fully leverage multiple processors on a given machine. It runs on both POSIX and Windows. -The :mod:`multiprocessing` module also introduces APIs which do not have -analogs in the :mod:`threading` module. A prime example of this is the +The :mod:`!multiprocessing` module also introduces the :class:`~multiprocessing.pool.Pool` object which offers a convenient means of parallelizing the execution of a function across multiple input values, distributing the input data across processes (data parallelism). The following @@ -44,6 +43,10 @@ will print to standard output :: [1, 4, 9] +The :mod:`!multiprocessing` module also introduces APIs which do not have +analogs in the :mod:`threading` module, like the ability to :meth:`terminate +`, :meth:`interrupt ` or :meth:`kill +` a running process. .. seealso:: @@ -58,7 +61,7 @@ will print to standard output :: The :class:`Process` class ^^^^^^^^^^^^^^^^^^^^^^^^^^ -In :mod:`multiprocessing`, processes are spawned by creating a :class:`Process` +In :mod:`!multiprocessing`, processes are spawned by creating a :class:`Process` object and then calling its :meth:`~Process.start` method. :class:`Process` follows the API of :class:`threading.Thread`. A trivial example of a multiprocess program is :: @@ -97,6 +100,10 @@ To show the individual process IDs involved, here is an expanded example:: For an explanation of why the ``if __name__ == '__main__'`` part is necessary, see :ref:`multiprocessing-programming`. +The arguments to :class:`Process` usually need to be picklable so they can be +passed to the child process. If you tried typing the above example directly +into a REPL it could lead to an :exc:`AttributeError` in the child process +trying to locate the *f* function in the ``__main__`` module. .. _multiprocessing-start-methods: @@ -104,7 +111,7 @@ necessary, see :ref:`multiprocessing-programming`. Contexts and start methods ^^^^^^^^^^^^^^^^^^^^^^^^^^ -Depending on the platform, :mod:`multiprocessing` supports three ways +Depending on the platform, :mod:`!multiprocessing` supports three ways to start a process. These *start methods* are .. _multiprocessing-start-method-spawn: @@ -233,9 +240,12 @@ processes for a different context. In particular, locks created using the *fork* context cannot be passed to processes started using the *spawn* or *forkserver* start methods. -A library which wants to use a particular start method should probably -use :func:`get_context` to avoid interfering with the choice of the -library user. +Libraries using :mod:`!multiprocessing` or +:class:`~concurrent.futures.ProcessPoolExecutor` should be designed to allow +their users to provide their own multiprocessing context. Using a specific +context of your own within a library can lead to incompatibilities with the +rest of the library user's application. Always document if your library +requires a specific start method. .. warning:: @@ -248,7 +258,7 @@ library user. Exchanging objects between processes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`multiprocessing` supports two types of communication channel between +:mod:`!multiprocessing` supports two types of communication channel between processes: **Queues** @@ -269,7 +279,7 @@ processes: p.join() Queues are thread and process safe. - Any object put into a :mod:`~multiprocessing` queue will be serialized. + Any object put into a :mod:`!multiprocessing` queue will be serialized. **Pipes** @@ -303,7 +313,7 @@ processes: Synchronization between processes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`multiprocessing` contains equivalents of all the synchronization +:mod:`!multiprocessing` contains equivalents of all the synchronization primitives from :mod:`threading`. For instance one can use a lock to ensure that only one process prints to standard output at a time:: @@ -334,7 +344,7 @@ avoid using shared state as far as possible. This is particularly true when using multiple processes. However, if you really do need to use some shared data then -:mod:`multiprocessing` provides a couple of ways of doing so. +:mod:`!multiprocessing` provides a couple of ways of doing so. **Shared memory** @@ -508,9 +518,24 @@ process which created it. Reference --------- -The :mod:`multiprocessing` package mostly replicates the API of the +The :mod:`!multiprocessing` package mostly replicates the API of the :mod:`threading` module. +.. _global-start-method: + +Global start method +^^^^^^^^^^^^^^^^^^^ + +Python supports several ways to create and initialize a process. +The global start method sets the default mechanism for creating a process. + +Several multiprocessing functions and methods that may also instantiate +certain objects will implicitly set the global start method to the system's default, +if it hasn’t been set already. The global start method can only be set once. +If you need to change the start method from the system default, you must +proactively set the global start method before calling functions or methods, +or creating these objects. + :class:`Process` and exceptions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -538,9 +563,42 @@ The :mod:`multiprocessing` package mostly replicates the API of the to pass to *target*. If a subclass overrides the constructor, it must make sure it invokes the - base class constructor (:meth:`Process.__init__`) before doing anything else + base class constructor (``super().__init__()``) before doing anything else to the process. + .. note:: + + In general, all arguments to :class:`Process` must be picklable. This is + frequently observed when trying to create a :class:`Process` or use a + :class:`concurrent.futures.ProcessPoolExecutor` from a REPL with a + locally defined *target* function. + + Passing a callable object defined in the current REPL session causes the + child process to die via an uncaught :exc:`AttributeError` exception when + starting as *target* must have been defined within an importable module + in order to be loaded during unpickling. + + Example of this uncatchable error from the child:: + + >>> import multiprocessing as mp + >>> def knigit(): + ... print("Ni!") + ... + >>> process = mp.Process(target=knigit) + >>> process.start() + >>> Traceback (most recent call last): + File ".../multiprocessing/spawn.py", line ..., in spawn_main + File ".../multiprocessing/spawn.py", line ..., in _main + AttributeError: module '__main__' has no attribute 'knigit' + >>> process + + + See :ref:`multiprocessing-programming-spawn`. While this restriction is + not true if using the ``"fork"`` start method, as of Python ``3.14`` that + is no longer the default on any platform. See + :ref:`multiprocessing-start-methods`. + See also :gh:`132898`. + .. versionchanged:: 3.3 Added the *daemon* parameter. @@ -646,7 +704,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the The process's authentication key (a byte string). - When :mod:`multiprocessing` is initialized the main process is assigned a + When :mod:`!multiprocessing` is initialized the main process is assigned a random string using :func:`os.urandom`. When a :class:`Process` object is created, it will inherit the @@ -747,7 +805,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the .. exception:: ProcessError - The base class of all :mod:`multiprocessing` exceptions. + The base class of all :mod:`!multiprocessing` exceptions. .. exception:: BufferTooShort @@ -787,19 +845,19 @@ If you use :class:`JoinableQueue` then you **must** call semaphore used to count the number of unfinished tasks may eventually overflow, raising an exception. -One difference from other Python queue implementations, is that :mod:`multiprocessing` +One difference from other Python queue implementations, is that :mod:`!multiprocessing` queues serializes all objects that are put into them using :mod:`pickle`. -The object return by the get method is a re-created object that does not share memory -with the original object. +The object returned by the get method is a re-created object that does not share +memory with the original object. Note that one can also create a shared queue by using a manager object -- see :ref:`multiprocessing-managers`. .. note:: - :mod:`multiprocessing` uses the usual :exc:`queue.Empty` and + :mod:`!multiprocessing` uses the usual :exc:`queue.Empty` and :exc:`queue.Full` exceptions to signal a timeout. They are not available in - the :mod:`multiprocessing` namespace so you need to import them from + the :mod:`!multiprocessing` namespace so you need to import them from :mod:`queue`. .. note:: @@ -847,7 +905,7 @@ For an example of the usage of queues for interprocess communication see :ref:`multiprocessing-examples`. -.. function:: Pipe([duplex]) +.. function:: Pipe(duplex=True) Returns a pair ``(conn1, conn2)`` of :class:`~multiprocessing.connection.Connection` objects representing the @@ -867,11 +925,15 @@ For an example of the usage of queues for interprocess communication see locks/semaphores. When a process first puts an item on the queue a feeder thread is started which transfers objects from a buffer into the pipe. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + The usual :exc:`queue.Empty` and :exc:`queue.Full` exceptions from the standard library's :mod:`queue` module are raised to signal timeouts. :class:`Queue` implements all the methods of :class:`queue.Queue` except for - :meth:`~queue.Queue.task_done` and :meth:`~queue.Queue.join`. + :meth:`~queue.Queue.task_done`, :meth:`~queue.Queue.join`, and + :meth:`~queue.Queue.shutdown`. .. method:: qsize() @@ -982,6 +1044,9 @@ For an example of the usage of queues for interprocess communication see It is a simplified :class:`Queue` type, very close to a locked :class:`Pipe`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + .. method:: close() Close the queue: release internal resources. @@ -1012,6 +1077,9 @@ For an example of the usage of queues for interprocess communication see :class:`JoinableQueue`, a :class:`Queue` subclass, is a queue which additionally has :meth:`task_done` and :meth:`join` methods. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + .. method:: task_done() Indicate that a formerly enqueued task is complete. Used by queue @@ -1085,7 +1153,7 @@ Miscellaneous .. function:: freeze_support() - Add support for when a program which uses :mod:`multiprocessing` has been + Add support for when a program which uses :mod:`!multiprocessing` has been frozen to produce an executable. (Has been tested with **py2exe**, **PyInstaller** and **cx_Freeze**.) @@ -1121,11 +1189,11 @@ Miscellaneous .. function:: get_context(method=None) Return a context object which has the same attributes as the - :mod:`multiprocessing` module. + :mod:`!multiprocessing` module. If *method* is ``None`` then the default context is returned. Note that if - the global start method has not been set, this will set it to the - default method. + the global start method has not been set, this will set it to the system default + See :ref:`global-start-method` for more details. Otherwise *method* should be ``'fork'``, ``'spawn'``, ``'forkserver'``. :exc:`ValueError` is raised if the specified start method is not available. See :ref:`multiprocessing-start-methods`. @@ -1136,10 +1204,9 @@ Miscellaneous Return the name of start method used for starting processes. - If the global start method has not been set and *allow_none* is - ``False``, then the start method is set to the default and the name - is returned. If the start method has not been set and *allow_none* is - ``True`` then ``None`` is returned. + If the global start method is not set and *allow_none* is ``False``, the global start + method is set to the default, and its name is returned. See + :ref:`global-start-method` for more details. The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'`` or ``None``. See :ref:`multiprocessing-start-methods`. @@ -1156,7 +1223,7 @@ Miscellaneous Set the path of the Python interpreter to use when starting a child process. (By default :data:`sys.executable` is used). Embedders will probably need to - do some thing like :: + do something like :: set_executable(os.path.join(sys.exec_prefix, 'pythonw.exe')) @@ -1168,22 +1235,32 @@ Miscellaneous .. versionchanged:: 3.11 Accepts a :term:`path-like object`. -.. function:: set_forkserver_preload(module_names) +.. function:: set_forkserver_preload(module_names, *, on_error='ignore') Set a list of module names for the forkserver main process to attempt to import so that their already imported state is inherited by forked - processes. Any :exc:`ImportError` when doing so is silently ignored. - This can be used as a performance enhancement to avoid repeated work - in every process. + processes. This can be used as a performance enhancement to avoid repeated + work in every process. For this to work, it must be called before the forkserver process has been launched (before creating a :class:`Pool` or starting a :class:`Process`). + The *on_error* parameter controls how :exc:`ImportError` exceptions during + module preloading are handled: ``"ignore"`` (default) silently ignores + failures, ``"warn"`` causes the forkserver subprocess to emit an + :exc:`ImportWarning` to stderr, and ``"fail"`` causes the forkserver + subprocess to exit with the exception traceback on stderr, making + subsequent process creation fail with :exc:`EOFError` or + :exc:`ConnectionError`. + Only meaningful when using the ``'forkserver'`` start method. See :ref:`multiprocessing-start-methods`. .. versionadded:: 3.4 + .. versionchanged:: 3.15 + Added the *on_error* parameter. + .. function:: set_start_method(method, force=False) Set the method which should be used to start child processes. @@ -1203,7 +1280,7 @@ Miscellaneous .. note:: - :mod:`multiprocessing` contains no analogues of + :mod:`!multiprocessing` contains no analogues of :func:`threading.active_count`, :func:`threading.enumerate`, :func:`threading.settrace`, :func:`threading.setprofile`, :class:`threading.Timer`, or :class:`threading.local`. @@ -1259,12 +1336,12 @@ Connection objects are usually created using Note that multiple connection objects may be polled at once by using :func:`multiprocessing.connection.wait`. - .. method:: send_bytes(buffer[, offset[, size]]) + .. method:: send_bytes(buf[, offset[, size]]) Send byte data from a :term:`bytes-like object` as a complete message. - If *offset* is given then data is read from that position in *buffer*. If - *size* is given then that many bytes will be read from buffer. Very large + If *offset* is given then data is read from that position in *buf*. If + *size* is given then that many bytes will be read from *buf*. Very large buffers (approximately 32 MiB+, though it depends on the OS) may raise a :exc:`ValueError` exception @@ -1284,18 +1361,18 @@ Connection objects are usually created using alias of :exc:`OSError`. - .. method:: recv_bytes_into(buffer[, offset]) + .. method:: recv_bytes_into(buf[, offset]) - Read into *buffer* a complete message of byte data sent from the other end + Read into *buf* a complete message of byte data sent from the other end of the connection and return the number of bytes in the message. Blocks until there is something to receive. Raises :exc:`EOFError` if there is nothing left to receive and the other end was closed. - *buffer* must be a writable :term:`bytes-like object`. If + *buf* must be a writable :term:`bytes-like object`. If *offset* is given then the message will be written into the buffer from that position. Offset must be a non-negative integer less than the - length of *buffer* (in bytes). + length of *buf* (in bytes). If the buffer is too short then a :exc:`BufferTooShort` exception is raised and the complete message is available as ``e.args[0]`` where ``e`` @@ -1366,6 +1443,9 @@ object -- see :ref:`multiprocessing-managers`. A barrier object: a clone of :class:`threading.Barrier`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + .. versionadded:: 3.3 .. class:: BoundedSemaphore([value]) @@ -1373,6 +1453,9 @@ object -- see :ref:`multiprocessing-managers`. A bounded semaphore object: a close analog of :class:`threading.BoundedSemaphore`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + A solitary difference from its close analog exists: its ``acquire`` method's first argument is named *block*, as is consistent with :meth:`Lock.acquire`. @@ -1391,7 +1474,10 @@ object -- see :ref:`multiprocessing-managers`. A condition variable: an alias for :class:`threading.Condition`. If *lock* is specified then it should be a :class:`Lock` or :class:`RLock` - object from :mod:`multiprocessing`. + object from :mod:`!multiprocessing`. + + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. .. versionchanged:: 3.3 The :meth:`~threading.Condition.wait_for` method was added. @@ -1400,6 +1486,8 @@ object -- see :ref:`multiprocessing-managers`. A clone of :class:`threading.Event`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. .. class:: Lock() @@ -1415,6 +1503,9 @@ object -- see :ref:`multiprocessing-managers`. instance of ``multiprocessing.synchronize.Lock`` initialized with a default context. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + :class:`Lock` supports the :term:`context manager` protocol and thus may be used in :keyword:`with` statements. @@ -1472,6 +1563,9 @@ object -- see :ref:`multiprocessing-managers`. instance of ``multiprocessing.synchronize.RLock`` initialized with a default context. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + :class:`RLock` supports the :term:`context manager` protocol and thus may be used in :keyword:`with` statements. @@ -1531,15 +1625,28 @@ object -- see :ref:`multiprocessing-managers`. A semaphore object: a close analog of :class:`threading.Semaphore`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + A solitary difference from its close analog exists: its ``acquire`` method's first argument is named *block*, as is consistent with :meth:`Lock.acquire`. + + .. method:: get_value() + + Return the current value of semaphore. + + Note that this may raise :exc:`NotImplementedError` on platforms like + macOS where ``sem_getvalue()`` is not implemented. + + .. method:: locked() Return a boolean indicating whether this object is locked right now. .. versionadded:: 3.14 + .. note:: On macOS, ``sem_timedwait`` is unsupported, so calling ``acquire()`` with @@ -1597,11 +1704,14 @@ inherited by child processes. value is actually a synchronized wrapper for the array. *typecode_or_type* determines the type of the elements of the returned array: - it is either a ctypes type or a one character typecode of the kind used by - the :mod:`array` module. If *size_or_initializer* is an integer, then it - determines the length of the array, and the array will be initially zeroed. - Otherwise, *size_or_initializer* is a sequence which is used to initialize - the array and whose length determines the length of the array. + it is either a :ref:`ctypes type ` or a one + character typecode of the kind used by the :mod:`array` module with the + exception of ``'w'``, which is not supported. In addition, the ``'c'`` + typecode is an alias for :class:`ctypes.c_char`. If *size_or_initializer* + is an integer, then it determines the length of the array, and the array + will be initially zeroed. Otherwise, *size_or_initializer* is a sequence + which is used to initialize the array and whose length determines the length + of the array. If *lock* is ``True`` (the default) then a new lock object is created to synchronize access to the value. If *lock* is a :class:`Lock` or @@ -1613,16 +1723,19 @@ inherited by child processes. Note that *lock* is a keyword only argument. Note that an array of :data:`ctypes.c_char` has *value* and *raw* - attributes which allow one to use it to store and retrieve strings. + attributes which can both be used to store and retrieve byte strings. + While *raw* allows interaction with a :class:`bytes` object the full size of + the array, reading *value* will terminate after a null byte, like most + programming languages handle strings. -The :mod:`multiprocessing.sharedctypes` module -"""""""""""""""""""""""""""""""""""""""""""""" +The :mod:`!multiprocessing.sharedctypes` module +""""""""""""""""""""""""""""""""""""""""""""""" .. module:: multiprocessing.sharedctypes :synopsis: Allocate ctypes objects from shared memory. -The :mod:`multiprocessing.sharedctypes` module provides functions for allocating +The :mod:`!multiprocessing.sharedctypes` module provides functions for allocating :mod:`ctypes` objects from shared memory which can be inherited by child processes. @@ -1665,7 +1778,7 @@ processes. attributes which allow one to use it to store and retrieve strings -- see documentation for :mod:`ctypes`. -.. function:: Array(typecode_or_type, size_or_initializer, *, lock=True) +.. function:: Array(typecode_or_type, size_or_initializer, *, lock=True, ctx=None) The same as :func:`RawArray` except that depending on the value of *lock* a process-safe synchronization wrapper may be returned instead of a raw ctypes @@ -1679,9 +1792,13 @@ processes. automatically protected by a lock, so it will not necessarily be "process-safe". - Note that *lock* is a keyword-only argument. + *ctx* is a context object, or ``None`` (use the current context). If ``None``, + calling this may set the global start method. See + :ref:`global-start-method` for more details. -.. function:: Value(typecode_or_type, *args, lock=True) + Note that *lock* and *ctx* are keyword-only parameters. + +.. function:: Value(typecode_or_type, *args, lock=True, ctx=None) The same as :func:`RawValue` except that depending on the value of *lock* a process-safe synchronization wrapper may be returned instead of a raw ctypes @@ -1694,19 +1811,27 @@ processes. automatically protected by a lock, so it will not necessarily be "process-safe". - Note that *lock* is a keyword-only argument. + *ctx* is a context object, or ``None`` (use the current context). If ``None``, + calling this may set the global start method. See + :ref:`global-start-method` for more details. + + Note that *lock* and *ctx* are keyword-only parameters. .. function:: copy(obj) Return a ctypes object allocated from shared memory which is a copy of the ctypes object *obj*. -.. function:: synchronized(obj[, lock]) +.. function:: synchronized(obj, lock=None, ctx=None) Return a process-safe wrapper object for a ctypes object which uses *lock* to synchronize access. If *lock* is ``None`` (the default) then a :class:`multiprocessing.RLock` object is created automatically. + *ctx* is a context object, or ``None`` (use the current context). If ``None``, + calling this may set the global start method. See + :ref:`global-start-method` for more details. + A synchronized wrapper will have two methods in addition to those of the object it wraps: :meth:`get_obj` returns the wrapped object and :meth:`get_lock` returns the lock object used for synchronization. @@ -1824,8 +1949,9 @@ their parent process exits. The manager classes are defined in the *serializer* must be ``'pickle'`` (use :mod:`pickle` serialization) or ``'xmlrpclib'`` (use :mod:`xmlrpc.client` serialization). - *ctx* is a context object, or ``None`` (use the current context). See the - :func:`get_context` function. + *ctx* is a context object, or ``None`` (use the current context). If ``None``, + calling this may set the global start method. See + :ref:`global-start-method` for more details. *shutdown_timeout* is a timeout in seconds used to wait until the process used by the manager completes in the :meth:`shutdown` method. If the @@ -2209,7 +2335,7 @@ demonstrates a level of control over the synchronization. .. note:: - The proxy types in :mod:`multiprocessing` do nothing to support comparisons + The proxy types in :mod:`!multiprocessing` do nothing to support comparisons by value. So, for instance, we have: .. doctest:: @@ -2318,7 +2444,9 @@ with the :class:`Pool` class. the worker processes. Usually a pool is created using the function :func:`multiprocessing.Pool` or the :meth:`Pool` method of a context object. In both cases *context* is set - appropriately. + appropriately. If ``None``, calling this function will have the side effect + of setting the current global start method if it has not been set already. + See the :func:`get_context` function. Note that the methods of the pool object should only be called by the process which created the pool. @@ -2349,7 +2477,7 @@ with the :class:`Pool` class. duration of the Pool's work queue. A frequent pattern found in other systems (such as Apache, mod_wsgi, etc) to free resources held by workers is to allow a worker within a pool to complete only a set - amount of work before being exiting, being cleaned up and a new + amount of work before exiting, being cleaned up and a new process spawned to replace the old one. The *maxtasksperchild* argument to the :class:`Pool` exposes this ability to the end user. @@ -2534,7 +2662,7 @@ Usually message passing between processes is done using queues or by using :class:`~Connection` objects returned by :func:`~multiprocessing.Pipe`. -However, the :mod:`multiprocessing.connection` module allows some extra +However, the :mod:`!multiprocessing.connection` module allows some extra flexibility. It basically gives a high level message oriented API for dealing with sockets or Windows named pipes. It also has support for *digest authentication* using the :mod:`hmac` module, and for polling @@ -2792,6 +2920,16 @@ between themselves. Suitable authentication keys can also be generated by using :func:`os.urandom`. +This authentication protects :class:`Listener` and :func:`Client` connections, +which are reachable by address. It is not applied to the anonymous pipes +created by :func:`~multiprocessing.Pipe` or used internally by +:class:`~multiprocessing.Queue`. +:mod:`multiprocessing` treats all local processes running as the same user as +trusted; on most operating systems such processes can access each other's pipe +file descriptors regardless. Applications that require isolation between +processes of the same user must arrange it at the operating-system level -- +for example, by running workers under a different user account or in a sandbox. + Logging ^^^^^^^ @@ -2803,7 +2941,7 @@ handler type) for messages from different processes to get mixed up. .. currentmodule:: multiprocessing .. function:: get_logger() - Returns the logger used by :mod:`multiprocessing`. If necessary, a new one + Returns the logger used by :mod:`!multiprocessing`. If necessary, a new one will be created. When first created the logger has level :const:`logging.NOTSET` and no @@ -2841,18 +2979,18 @@ Below is an example session with logging turned on:: For a full table of logging levels, see the :mod:`logging` module. -The :mod:`multiprocessing.dummy` module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The :mod:`!multiprocessing.dummy` module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: multiprocessing.dummy :synopsis: Dumb wrapper around threading. -:mod:`multiprocessing.dummy` replicates the API of :mod:`multiprocessing` but is +:mod:`!multiprocessing.dummy` replicates the API of :mod:`!multiprocessing` but is no more than a wrapper around the :mod:`threading` module. .. currentmodule:: multiprocessing.pool -In particular, the ``Pool`` function provided by :mod:`multiprocessing.dummy` +In particular, the ``Pool`` function provided by :mod:`!multiprocessing.dummy` returns an instance of :class:`ThreadPool`, which is a subclass of :class:`Pool` that supports all the same method calls but uses a pool of worker threads rather than worker processes. @@ -2897,7 +3035,7 @@ Programming guidelines ---------------------- There are certain guidelines and idioms which should be adhered to when using -:mod:`multiprocessing`. +:mod:`!multiprocessing`. All start methods @@ -2938,7 +3076,7 @@ Joining zombie processes Better to inherit than pickle/unpickle When using the *spawn* or *forkserver* start methods many types - from :mod:`multiprocessing` need to be picklable so that child + from :mod:`!multiprocessing` need to be picklable so that child processes can use them. However, one should generally avoid sending shared objects to other processes using pipes or queues. Instead you should arrange the program so that a process which @@ -3028,7 +3166,7 @@ Explicitly pass resources to child processes Beware of replacing :data:`sys.stdin` with a "file like object" - :mod:`multiprocessing` originally unconditionally called:: + :mod:`!multiprocessing` originally unconditionally called:: os.close(sys.stdin.fileno()) @@ -3070,10 +3208,10 @@ start method. More picklability - Ensure that all arguments to :meth:`Process.__init__` are picklable. - Also, if you subclass :class:`~multiprocessing.Process` then make sure that - instances will be picklable when the :meth:`Process.start - ` method is called. + Ensure that all arguments to :class:`~multiprocessing.Process` are + picklable. Also, if you subclass ``Process.__init__``, you must make sure + that instances will be picklable when the + :meth:`Process.start ` method is called. Global variables diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index 74c97e8c9a97598..3fbd2b57426ebe7 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -4,9 +4,6 @@ .. module:: netrc :synopsis: Loading of .netrc files. -.. moduleauthor:: Eric S. Raymond -.. sectionauthor:: Eric S. Raymond - **Source code:** :source:`Lib/netrc.py` -------------- diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index 681d0b76f2a14b7..57b35017072c97e 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -69,11 +69,11 @@ The numeric tower .. attribute:: numerator - Abstract. + Abstract. The numerator of this rational number. .. attribute:: denominator - Abstract. + Abstract. The denominator of this rational number. .. class:: Integral diff --git a/Doc/library/numeric.rst b/Doc/library/numeric.rst index 7c76a479d73b264..a2109a29bfb6cc5 100644 --- a/Doc/library/numeric.rst +++ b/Doc/library/numeric.rst @@ -19,6 +19,7 @@ The following modules are documented in this chapter: numbers.rst math.rst + math.integer.rst cmath.rst decimal.rst fractions.rst diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index e8e71068dd99ebc..3d1c8cda13be381 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -4,8 +4,6 @@ .. module:: operator :synopsis: Functions corresponding to the standard operators. -.. sectionauthor:: Skip Montanaro - **Source code:** :source:`Lib/operator.py` .. testsetup:: @@ -15,7 +13,7 @@ -------------- -The :mod:`operator` module exports a set of efficient functions corresponding to +The :mod:`!operator` module exports a set of efficient functions corresponding to the intrinsic operators of Python. For example, ``operator.add(x, y)`` is equivalent to the expression ``x+y``. Many function names are those used for special methods, without the double underscores. For backward compatibility, @@ -112,7 +110,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: and_(a, b) __and__(a, b) - Return the bitwise and of *a* and *b*. + Return ``a & b``. .. function:: floordiv(a, b) @@ -136,13 +134,13 @@ The mathematical and bitwise operations are the most numerous: __inv__(obj) __invert__(obj) - Return the bitwise inverse of the number *obj*. This is equivalent to ``~obj``. + Return ``~obj``. .. function:: lshift(a, b) __lshift__(a, b) - Return *a* shifted left by *b*. + Return ``a << b``. .. function:: mod(a, b) @@ -154,7 +152,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: mul(a, b) __mul__(a, b) - Return ``a * b``, for *a* and *b* numbers. + Return ``a * b``. .. function:: matmul(a, b) @@ -174,25 +172,25 @@ The mathematical and bitwise operations are the most numerous: .. function:: or_(a, b) __or__(a, b) - Return the bitwise or of *a* and *b*. + Return ``a | b``. .. function:: pos(obj) __pos__(obj) - Return *obj* positive (``+obj``). + Return ``+obj``. .. function:: pow(a, b) __pow__(a, b) - Return ``a ** b``, for *a* and *b* numbers. + Return ``a ** b``. .. function:: rshift(a, b) __rshift__(a, b) - Return *a* shifted right by *b*. + Return ``a >> b``. .. function:: sub(a, b) @@ -211,7 +209,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: xor(a, b) __xor__(a, b) - Return the bitwise exclusive or of *a* and *b*. + Return ``a ^ b``. Operations which work with sequences (some of them with mappings too) include: @@ -275,7 +273,7 @@ The following operation works with callables: .. versionadded:: 3.11 -The :mod:`operator` module also defines tools for generalized attribute and item +The :mod:`!operator` module also defines tools for generalized attribute and item lookups. These are useful for making fast field extractors as arguments for :func:`map`, :func:`sorted`, :meth:`itertools.groupby`, or other functions that expect a function argument. @@ -390,7 +388,7 @@ Mapping Operators to Functions ------------------------------ This table shows how abstract operations correspond to operator symbols in the -Python syntax and the functions in the :mod:`operator` module. +Python syntax and the functions in the :mod:`!operator` module. +-----------------------+-------------------------+---------------------------------------+ | Operation | Syntax | Function | @@ -405,13 +403,18 @@ Python syntax and the functions in the :mod:`operator` module. +-----------------------+-------------------------+---------------------------------------+ | Division | ``a // b`` | ``floordiv(a, b)`` | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise And | ``a & b`` | ``and_(a, b)`` | +| Bitwise And, or | ``a & b`` | ``and_(a, b)`` | +| Intersection | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Exclusive Or | ``a ^ b`` | ``xor(a, b)`` | +| Bitwise Exclusive Or, | ``a ^ b`` | ``xor(a, b)`` | +| or Symmetric | | | +| Difference | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Inversion | ``~ a`` | ``invert(a)`` | +| Bitwise Inversion, or | ``~ a`` | ``invert(a)`` | +| Complement | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Or | ``a | b`` | ``or_(a, b)`` | +| Bitwise Or, or | ``a | b`` | ``or_(a, b)`` | +| Union | | | +-----------------------+-------------------------+---------------------------------------+ | Exponentiation | ``a ** b`` | ``pow(a, b)`` | +-----------------------+-------------------------+---------------------------------------+ diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index ff327cf9162a8cc..905212965bd70fe 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -4,9 +4,6 @@ .. module:: optparse :synopsis: Command-line option parsing library. -.. moduleauthor:: Greg Ward -.. sectionauthor:: Greg Ward - **Source code:** :source:`Lib/optparse.py` -------------- @@ -20,7 +17,7 @@ The standard library includes three argument parsing libraries: * :mod:`getopt`: a module that closely mirrors the procedural C ``getopt`` API. Included in the standard library since before the initial Python 1.0 release. -* :mod:`optparse`: a declarative replacement for ``getopt`` that +* :mod:`!optparse`: a declarative replacement for ``getopt`` that provides equivalent functionality without requiring each application to implement its own procedural option parsing logic. Included in the standard library since the Python 2.3 release. @@ -37,10 +34,10 @@ the highest level of baseline functionality with the least application level cod However, it also serves a niche use case as a tool for prototyping and testing command line argument handling in ``getopt``-based C applications. -:mod:`optparse` should be considered as an alternative to :mod:`argparse` in the +:mod:`!optparse` should be considered as an alternative to :mod:`argparse` in the following cases: -* an application is already using :mod:`optparse` and doesn't want to risk the +* an application is already using :mod:`!optparse` and doesn't want to risk the subtle behavioural changes that may arise when migrating to :mod:`argparse` * the application requires additional control over the way options and positional parameters are interleaved on the command line (including @@ -55,7 +52,7 @@ following cases: behavior which ``argparse`` does not support, but which can be implemented in terms of the lower level interface offered by ``optparse`` -These considerations also mean that :mod:`optparse` is likely to provide a +These considerations also mean that :mod:`!optparse` is likely to provide a better foundation for library authors writing third party command line argument processing libraries. @@ -126,15 +123,15 @@ application use case. Introduction ------------ -:mod:`optparse` is a more convenient, flexible, and powerful library for parsing +:mod:`!optparse` is a more convenient, flexible, and powerful library for parsing command-line options than the minimalist :mod:`getopt` module. -:mod:`optparse` uses a more declarative style of command-line parsing: +:mod:`!optparse` uses a more declarative style of command-line parsing: you create an instance of :class:`OptionParser`, populate it with options, and parse the command line. -:mod:`optparse` allows users to specify options in the conventional +:mod:`!optparse` allows users to specify options in the conventional GNU/POSIX syntax, and additionally generates usage and help messages for you. -Here's an example of using :mod:`optparse` in a simple script:: +Here's an example of using :mod:`!optparse` in a simple script:: from optparse import OptionParser ... @@ -152,11 +149,11 @@ on the command-line, for example:: --file=outfile -q -As it parses the command line, :mod:`optparse` sets attributes of the +As it parses the command line, :mod:`!optparse` sets attributes of the ``options`` object returned by :meth:`~OptionParser.parse_args` based on user-supplied command-line values. When :meth:`~OptionParser.parse_args` returns from parsing this command line, ``options.filename`` will be ``"outfile"`` and ``options.verbose`` will be -``False``. :mod:`optparse` supports both long and short options, allows short +``False``. :mod:`!optparse` supports both long and short options, allows short options to be merged together, and allows options to be associated with their arguments in a variety of ways. Thus, the following command lines are all equivalent to the above example:: @@ -171,7 +168,7 @@ Additionally, users can run one of the following :: -h --help -and :mod:`optparse` will print out a brief summary of your script's options: +and :mod:`!optparse` will print out a brief summary of your script's options: .. code-block:: text @@ -191,7 +188,7 @@ where the value of *yourscript* is determined at runtime (normally from Background ---------- -:mod:`optparse` was explicitly designed to encourage the creation of programs +:mod:`!optparse` was explicitly designed to encourage the creation of programs with straightforward command-line interfaces that follow the conventions established by the :c:func:`!getopt` family of functions available to C developers. To that end, it supports only the most common command-line syntax and semantics @@ -223,7 +220,7 @@ option options to be merged into a single argument, e.g. ``-x -F`` is equivalent to ``-xF``. The GNU project introduced ``--`` followed by a series of hyphen-separated words, e.g. ``--file`` or ``--dry-run``. These are the - only two option syntaxes provided by :mod:`optparse`. + only two option syntaxes provided by :mod:`!optparse`. Some other option syntaxes that the world has seen include: @@ -240,7 +237,7 @@ option * a slash followed by a letter, or a few letters, or a word, e.g. ``/f``, ``/file`` - These option syntaxes are not supported by :mod:`optparse`, and they never + These option syntaxes are not supported by :mod:`!optparse`, and they never will be. This is deliberate: the first three are non-standard on any environment, and the last only makes sense if you're exclusively targeting Windows or certain legacy platforms (e.g. VMS, MS-DOS). @@ -248,7 +245,7 @@ option option argument an argument that follows an option, is closely associated with that option, and is consumed from the argument list when that option is. With - :mod:`optparse`, option arguments may either be in a separate argument from + :mod:`!optparse`, option arguments may either be in a separate argument from their option: .. code-block:: text @@ -268,7 +265,7 @@ option argument will take an argument if they see it, and won't if they don't. This is somewhat controversial, because it makes parsing ambiguous: if ``-a`` takes an optional argument and ``-b`` is another option entirely, how do we - interpret ``-ab``? Because of this ambiguity, :mod:`optparse` does not + interpret ``-ab``? Because of this ambiguity, :mod:`!optparse` does not support this feature. positional argument @@ -278,7 +275,7 @@ positional argument required option an option that must be supplied on the command-line; note that the phrase - "required option" is self-contradictory in English. :mod:`optparse` doesn't + "required option" is self-contradictory in English. :mod:`!optparse` doesn't prevent you from implementing required options, but doesn't give you much help at it either. @@ -357,9 +354,9 @@ too many options can overwhelm users and make your code much harder to maintain. Tutorial -------- -While :mod:`optparse` is quite flexible and powerful, it's also straightforward +While :mod:`!optparse` is quite flexible and powerful, it's also straightforward to use in most cases. This section covers the code patterns that are common to -any :mod:`optparse`\ -based program. +any :mod:`!optparse`\ -based program. First, you need to import the OptionParser class; then, early in the main program, create an OptionParser instance:: @@ -374,7 +371,7 @@ Then you can start defining options. The basic syntax is:: attr=value, ...) Each option has one or more option strings, such as ``-f`` or ``--file``, -and several option attributes that tell :mod:`optparse` what to expect and what +and several option attributes that tell :mod:`!optparse` what to expect and what to do when it encounters that option on the command line. Typically, each option will have one short option string and one long option @@ -389,10 +386,10 @@ string overall. The option strings passed to :meth:`OptionParser.add_option` are effectively labels for the option defined by that call. For brevity, we will frequently refer to -*encountering an option* on the command line; in reality, :mod:`optparse` +*encountering an option* on the command line; in reality, :mod:`!optparse` encounters *option strings* and looks up options from them. -Once all of your options are defined, instruct :mod:`optparse` to parse your +Once all of your options are defined, instruct :mod:`!optparse` to parse your program's command line:: (options, args) = parser.parse_args() @@ -420,14 +417,14 @@ most fundamental. Understanding option actions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Actions tell :mod:`optparse` what to do when it encounters an option on the -command line. There is a fixed set of actions hard-coded into :mod:`optparse`; +Actions tell :mod:`!optparse` what to do when it encounters an option on the +command line. There is a fixed set of actions hard-coded into :mod:`!optparse`; adding new actions is an advanced topic covered in section -:ref:`optparse-extending-optparse`. Most actions tell :mod:`optparse` to store +:ref:`optparse-extending-optparse`. Most actions tell :mod:`!optparse` to store a value in some variable---for example, take a string from the command line and store it in an attribute of ``options``. -If you don't specify an option action, :mod:`optparse` defaults to ``store``. +If you don't specify an option action, :mod:`!optparse` defaults to ``store``. .. _optparse-store-action: @@ -435,7 +432,7 @@ If you don't specify an option action, :mod:`optparse` defaults to ``store``. The store action ^^^^^^^^^^^^^^^^ -The most common option action is ``store``, which tells :mod:`optparse` to take +The most common option action is ``store``, which tells :mod:`!optparse` to take the next argument (or the remainder of the current argument), ensure that it is of the correct type, and store it to your chosen destination. @@ -444,16 +441,16 @@ For example:: parser.add_option("-f", "--file", action="store", type="string", dest="filename") -Now let's make up a fake command line and ask :mod:`optparse` to parse it:: +Now let's make up a fake command line and ask :mod:`!optparse` to parse it:: args = ["-f", "foo.txt"] (options, args) = parser.parse_args(args) -When :mod:`optparse` sees the option string ``-f``, it consumes the next +When :mod:`!optparse` sees the option string ``-f``, it consumes the next argument, ``foo.txt``, and stores it in ``options.filename``. So, after this call to :meth:`~OptionParser.parse_args`, ``options.filename`` is ``"foo.txt"``. -Some other option types supported by :mod:`optparse` are ``int`` and ``float``. +Some other option types supported by :mod:`!optparse` are ``int`` and ``float``. Here's an option that expects an integer argument:: parser.add_option("-n", type="int", dest="num") @@ -470,19 +467,19 @@ right up against the option: since ``-n42`` (one argument) is equivalent to will print ``42``. -If you don't specify a type, :mod:`optparse` assumes ``string``. Combined with +If you don't specify a type, :mod:`!optparse` assumes ``string``. Combined with the fact that the default action is ``store``, that means our first example can be a lot shorter:: parser.add_option("-f", "--file", dest="filename") -If you don't supply a destination, :mod:`optparse` figures out a sensible +If you don't supply a destination, :mod:`!optparse` figures out a sensible default from the option strings: if the first long option string is ``--foo-bar``, then the default destination is ``foo_bar``. If there are no -long option strings, :mod:`optparse` looks at the first short option string: the +long option strings, :mod:`!optparse` looks at the first short option string: the default destination for ``-f`` is ``f``. -:mod:`optparse` also includes the built-in ``complex`` type. Adding +:mod:`!optparse` also includes the built-in ``complex`` type. Adding types is covered in section :ref:`optparse-extending-optparse`. @@ -492,7 +489,7 @@ Handling boolean (flag) options ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Flag options---set a variable to true or false when a particular option is -seen---are quite common. :mod:`optparse` supports them with two separate actions, +seen---are quite common. :mod:`!optparse` supports them with two separate actions, ``store_true`` and ``store_false``. For example, you might have a ``verbose`` flag that is turned on with ``-v`` and off with ``-q``:: @@ -503,7 +500,7 @@ Here we have two different options with the same destination, which is perfectly OK. (It just means you have to be a bit careful when setting default values---see below.) -When :mod:`optparse` encounters ``-v`` on the command line, it sets +When :mod:`!optparse` encounters ``-v`` on the command line, it sets ``options.verbose`` to ``True``; when it encounters ``-q``, ``options.verbose`` is set to ``False``. @@ -513,7 +510,7 @@ When :mod:`optparse` encounters ``-v`` on the command line, it sets Other actions ^^^^^^^^^^^^^ -Some other actions supported by :mod:`optparse` are: +Some other actions supported by :mod:`!optparse` are: ``"store_const"`` store a constant value, pre-set via :attr:`Option.const` @@ -539,11 +536,11 @@ Default values All of the above examples involve setting some variable (the "destination") when certain command-line options are seen. What happens if those options are never seen? Since we didn't supply any defaults, they are all set to ``None``. This -is usually fine, but sometimes you want more control. :mod:`optparse` lets you +is usually fine, but sometimes you want more control. :mod:`!optparse` lets you supply a default value for each destination, which is assigned before the command line is parsed. -First, consider the verbose/quiet example. If we want :mod:`optparse` to set +First, consider the verbose/quiet example. If we want :mod:`!optparse` to set ``verbose`` to ``True`` unless ``-q`` is seen, then we can do this:: parser.add_option("-v", action="store_true", dest="verbose", default=True) @@ -582,7 +579,7 @@ values, not both. Generating help ^^^^^^^^^^^^^^^ -:mod:`optparse`'s ability to generate help and usage text automatically is +:mod:`!optparse`'s ability to generate help and usage text automatically is useful for creating user-friendly command-line interfaces. All you have to do is supply a :attr:`~Option.help` value for each option, and optionally a short usage message for your whole program. Here's an OptionParser populated with @@ -603,7 +600,7 @@ user-friendly (documented) options:: help="interaction mode: novice, intermediate, " "or expert [default: %default]") -If :mod:`optparse` encounters either ``-h`` or ``--help`` on the +If :mod:`!optparse` encounters either ``-h`` or ``--help`` on the command-line, or if you just call :meth:`parser.print_help`, it prints the following to standard output: @@ -620,26 +617,26 @@ following to standard output: -m MODE, --mode=MODE interaction mode: novice, intermediate, or expert [default: intermediate] -(If the help output is triggered by a help option, :mod:`optparse` exits after +(If the help output is triggered by a help option, :mod:`!optparse` exits after printing the help text.) -There's a lot going on here to help :mod:`optparse` generate the best possible +There's a lot going on here to help :mod:`!optparse` generate the best possible help message: * the script defines its own usage message:: usage = "usage: %prog [options] arg1 arg2" - :mod:`optparse` expands ``%prog`` in the usage string to the name of the + :mod:`!optparse` expands ``%prog`` in the usage string to the name of the current program, i.e. ``os.path.basename(sys.argv[0])``. The expanded string is then printed before the detailed option help. - If you don't supply a usage string, :mod:`optparse` uses a bland but sensible + If you don't supply a usage string, :mod:`!optparse` uses a bland but sensible default: ``"Usage: %prog [options]"``, which is fine if your script doesn't take any positional arguments. * every option defines a help string, and doesn't worry about - line-wrapping---\ :mod:`optparse` takes care of wrapping lines and making + line-wrapping---\ :mod:`!optparse` takes care of wrapping lines and making the help output look good. * options that take a value indicate this fact in their automatically generated @@ -649,7 +646,7 @@ help message: Here, "MODE" is called the meta-variable: it stands for the argument that the user is expected to supply to ``-m``/``--mode``. By default, - :mod:`optparse` converts the destination variable name to uppercase and uses + :mod:`!optparse` converts the destination variable name to uppercase and uses that for the meta-variable. Sometimes, that's not what you want---for example, the ``--filename`` option explicitly sets ``metavar="FILE"``, resulting in this automatically generated option description:: @@ -663,7 +660,7 @@ help message: way to make your help text a lot clearer and more useful for end users. * options that have a default value can include ``%default`` in the help - string---\ :mod:`optparse` will replace it with :func:`str` of the option's + string---\ :mod:`!optparse` will replace it with :func:`str` of the option's default value. If an option has no default value (or the default value is ``None``), ``%default`` expands to ``none``. @@ -779,14 +776,14 @@ option groups is: Printing a version string ^^^^^^^^^^^^^^^^^^^^^^^^^ -Similar to the brief usage string, :mod:`optparse` can also print a version +Similar to the brief usage string, :mod:`!optparse` can also print a version string for your program. You have to supply the string as the ``version`` argument to OptionParser:: parser = OptionParser(usage="%prog [-f] [-q]", version="%prog 1.0") ``%prog`` is expanded just like it is in ``usage``. Apart from that, -``version`` can contain anything you like. When you supply it, :mod:`optparse` +``version`` can contain anything you like. When you supply it, :mod:`!optparse` automatically adds a ``--version`` option to your parser. If it encounters this option on the command line, it expands your ``version`` string (by replacing ``%prog``), prints it to stdout, and exits. @@ -815,10 +812,10 @@ The following two methods can be used to print and get the ``version`` string: .. _optparse-how-optparse-handles-errors: -How :mod:`optparse` handles errors -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +How :mod:`!optparse` handles errors +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -There are two broad classes of errors that :mod:`optparse` has to worry about: +There are two broad classes of errors that :mod:`!optparse` has to worry about: programmer errors and user errors. Programmer errors are usually erroneous calls to :func:`OptionParser.add_option`, e.g. invalid option strings, unknown option attributes, missing option attributes, etc. These are dealt with in the @@ -826,7 +823,7 @@ usual way: raise an exception (either :exc:`optparse.OptionError` or :exc:`TypeError`) and let the program crash. Handling user errors is much more important, since they are guaranteed to happen -no matter how stable your code is. :mod:`optparse` can automatically detect +no matter how stable your code is. :mod:`!optparse` can automatically detect some user errors, such as bad option arguments (passing ``-n 4x`` where ``-n`` takes an integer argument), missing arguments (``-n`` at the end of the command line, where ``-n`` takes an argument of any type). Also, @@ -838,7 +835,7 @@ condition:: if options.a and options.b: parser.error("options -a and -b are mutually exclusive") -In either case, :mod:`optparse` handles the error the same way: it prints the +In either case, :mod:`!optparse` handles the error the same way: it prints the program's usage message and an error message to standard error and exits with error status 2. @@ -861,11 +858,11 @@ Or, where the user fails to pass a value at all: foo: error: -n option requires an argument -:mod:`optparse`\ -generated error messages take care always to mention the +:mod:`!optparse`\ -generated error messages take care always to mention the option involved in the error; be sure to do the same when calling :func:`OptionParser.error` from your application code. -If :mod:`optparse`'s default error-handling behaviour does not suit your needs, +If :mod:`!optparse`'s default error-handling behaviour does not suit your needs, you'll need to subclass OptionParser and override its :meth:`~OptionParser.exit` and/or :meth:`~OptionParser.error` methods. @@ -875,7 +872,7 @@ and/or :meth:`~OptionParser.error` methods. Putting it all together ^^^^^^^^^^^^^^^^^^^^^^^ -Here's what :mod:`optparse`\ -based scripts usually look like:: +Here's what :mod:`!optparse`\ -based scripts usually look like:: from optparse import OptionParser ... @@ -911,7 +908,7 @@ Reference Guide Creating the parser ^^^^^^^^^^^^^^^^^^^ -The first step in using :mod:`optparse` is to create an OptionParser instance. +The first step in using :mod:`!optparse` is to create an OptionParser instance. .. class:: OptionParser(...) @@ -921,7 +918,7 @@ The first step in using :mod:`optparse` is to create an OptionParser instance. ``usage`` (default: ``"%prog [options]"``) The usage summary to print when your program is run incorrectly or with a - help option. When :mod:`optparse` prints the usage string, it expands + help option. When :mod:`!optparse` prints the usage string, it expands ``%prog`` to ``os.path.basename(sys.argv[0])`` (or to ``prog`` if you passed that keyword argument). To suppress a usage message, pass the special value :const:`optparse.SUPPRESS_USAGE`. @@ -938,7 +935,7 @@ The first step in using :mod:`optparse` is to create an OptionParser instance. ``version`` (default: ``None``) A version string to print when the user supplies a version option. If you - supply a true value for ``version``, :mod:`optparse` automatically adds a + supply a true value for ``version``, :mod:`!optparse` automatically adds a version option with the single option string ``--version``. The substring ``%prog`` is expanded the same as for ``usage``. @@ -949,17 +946,17 @@ The first step in using :mod:`optparse` is to create an OptionParser instance. ``description`` (default: ``None``) A paragraph of text giving a brief overview of your program. - :mod:`optparse` reformats this paragraph to fit the current terminal width + :mod:`!optparse` reformats this paragraph to fit the current terminal width and prints it when the user requests help (after ``usage``, but before the list of options). ``formatter`` (default: a new :class:`IndentedHelpFormatter`) An instance of optparse.HelpFormatter that will be used for printing help - text. :mod:`optparse` provides two concrete classes for this purpose: + text. :mod:`!optparse` provides two concrete classes for this purpose: IndentedHelpFormatter and TitledHelpFormatter. ``add_help_option`` (default: ``True``) - If true, :mod:`optparse` will add a help option (with option strings ``-h`` + If true, :mod:`!optparse` will add a help option (with option strings ``-h`` and ``--help``) to the parser. ``prog`` @@ -997,7 +994,7 @@ the OptionParser constructor, as in:: (:func:`make_option` is a factory function for creating Option instances; currently it is an alias for the Option constructor. A future version of -:mod:`optparse` may split Option into several classes, and :func:`make_option` +:mod:`!optparse` may split Option into several classes, and :func:`make_option` will pick the right class to instantiate. Do not instantiate Option directly.) @@ -1027,12 +1024,12 @@ The canonical way to create an :class:`Option` instance is with the The keyword arguments define attributes of the new Option object. The most important option attribute is :attr:`~Option.action`, and it largely determines which other attributes are relevant or required. If you pass - irrelevant option attributes, or fail to pass required ones, :mod:`optparse` + irrelevant option attributes, or fail to pass required ones, :mod:`!optparse` raises an :exc:`OptionError` exception explaining your mistake. - An option's *action* determines what :mod:`optparse` does when it encounters + An option's *action* determines what :mod:`!optparse` does when it encounters this option on the command-line. The standard option actions hard-coded into - :mod:`optparse` are: + :mod:`!optparse` are: ``"store"`` store this option's argument (default) @@ -1066,7 +1063,7 @@ The canonical way to create an :class:`Option` instance is with the attributes; see :ref:`optparse-standard-option-actions`.) As you can see, most actions involve storing or updating a value somewhere. -:mod:`optparse` always creates a special object for this, conventionally called +:mod:`!optparse` always creates a special object for this, conventionally called ``options``, which is an instance of :class:`optparse.Values`. .. class:: Values @@ -1084,7 +1081,7 @@ For example, when you call :: parser.parse_args() -one of the first things :mod:`optparse` does is create the ``options`` object:: +one of the first things :mod:`!optparse` does is create the ``options`` object:: options = Values() @@ -1099,7 +1096,7 @@ and the command-line being parsed includes any of the following:: --file=foo --file foo -then :mod:`optparse`, on seeing this option, will do the equivalent of :: +then :mod:`!optparse`, on seeing this option, will do the equivalent of :: options.filename = "foo" @@ -1124,13 +1121,13 @@ Option attributes The following option attributes may be passed as keyword arguments to :meth:`OptionParser.add_option`. If you pass an option attribute that is not relevant to a particular option, or fail to pass a required option attribute, -:mod:`optparse` raises :exc:`OptionError`. +:mod:`!optparse` raises :exc:`OptionError`. .. attribute:: Option.action (default: ``"store"``) - Determines :mod:`optparse`'s behaviour when this option is seen on the + Determines :mod:`!optparse`'s behaviour when this option is seen on the command line; the available options are documented :ref:`here `. @@ -1147,8 +1144,8 @@ relevant to a particular option, or fail to pass a required option attribute, (default: derived from option strings) If the option's action implies writing or modifying a value somewhere, this - tells :mod:`optparse` where to write it: :attr:`~Option.dest` names an - attribute of the ``options`` object that :mod:`optparse` builds as it parses + tells :mod:`!optparse` where to write it: :attr:`~Option.dest` names an + attribute of the ``options`` object that :mod:`!optparse` builds as it parses the command line. .. attribute:: Option.default @@ -1161,7 +1158,7 @@ relevant to a particular option, or fail to pass a required option attribute, (default: 1) How many arguments of type :attr:`~Option.type` should be consumed when this - option is seen. If > 1, :mod:`optparse` will store a tuple of values to + option is seen. If > 1, :mod:`!optparse` will store a tuple of values to :attr:`~Option.dest`. .. attribute:: Option.const @@ -1207,7 +1204,7 @@ Standard option actions The various option actions all have slightly different requirements and effects. Most actions have several relevant option attributes which you may specify to -guide :mod:`optparse`'s behaviour; a few have required attributes, which you +guide :mod:`!optparse`'s behaviour; a few have required attributes, which you must specify for any option using that action. * ``"store"`` [relevant: :attr:`~Option.type`, :attr:`~Option.dest`, @@ -1225,9 +1222,9 @@ must specify for any option using that action. If :attr:`~Option.type` is not supplied, it defaults to ``"string"``. - If :attr:`~Option.dest` is not supplied, :mod:`optparse` derives a destination + If :attr:`~Option.dest` is not supplied, :mod:`!optparse` derives a destination from the first long option string (e.g., ``--foo-bar`` implies - ``foo_bar``). If there are no long option strings, :mod:`optparse` derives a + ``foo_bar``). If there are no long option strings, :mod:`!optparse` derives a destination from the first short option string (e.g., ``-f`` implies ``f``). Example:: @@ -1239,7 +1236,7 @@ must specify for any option using that action. -f foo.txt -p 1 -3.5 4 -fbar.txt - :mod:`optparse` will set :: + :mod:`!optparse` will set :: options.f = "foo.txt" options.point = (1.0, -3.5, 4.0) @@ -1259,7 +1256,7 @@ must specify for any option using that action. parser.add_option("--noisy", action="store_const", const=2, dest="verbose") - If ``--noisy`` is seen, :mod:`optparse` will set :: + If ``--noisy`` is seen, :mod:`!optparse` will set :: options.verbose = 2 @@ -1282,7 +1279,7 @@ must specify for any option using that action. The option must be followed by an argument, which is appended to the list in :attr:`~Option.dest`. If no default value for :attr:`~Option.dest` is - supplied, an empty list is automatically created when :mod:`optparse` first + supplied, an empty list is automatically created when :mod:`!optparse` first encounters this option on the command-line. If :attr:`~Option.nargs` > 1, multiple arguments are consumed, and a tuple of length :attr:`~Option.nargs` is appended to :attr:`~Option.dest`. @@ -1294,7 +1291,7 @@ must specify for any option using that action. parser.add_option("-t", "--tracks", action="append", type="int") - If ``-t3`` is seen on the command-line, :mod:`optparse` does the equivalent + If ``-t3`` is seen on the command-line, :mod:`!optparse` does the equivalent of:: options.tracks = [] @@ -1333,7 +1330,7 @@ must specify for any option using that action. parser.add_option("-v", action="count", dest="verbosity") - The first time ``-v`` is seen on the command line, :mod:`optparse` does the + The first time ``-v`` is seen on the command line, :mod:`!optparse` does the equivalent of:: options.verbosity = 0 @@ -1364,7 +1361,7 @@ must specify for any option using that action. listed in the help message. To omit an option entirely, use the special value :const:`optparse.SUPPRESS_HELP`. - :mod:`optparse` automatically adds a :attr:`~Option.help` option to all + :mod:`!optparse` automatically adds a :attr:`~Option.help` option to all OptionParsers, so you do not normally need to create one. Example:: @@ -1382,7 +1379,7 @@ must specify for any option using that action. help="Input file to read data from") parser.add_option("--secret", help=SUPPRESS_HELP) - If :mod:`optparse` sees either ``-h`` or ``--help`` on the command line, + If :mod:`!optparse` sees either ``-h`` or ``--help`` on the command line, it will print something like the following help message to stdout (assuming ``sys.argv[0]`` is ``"foo.py"``): @@ -1395,7 +1392,7 @@ must specify for any option using that action. -v Be moderately verbose --file=FILENAME Input file to read data from - After printing the help message, :mod:`optparse` terminates your process with + After printing the help message, :mod:`!optparse` terminates your process with ``sys.exit(0)``. * ``"version"`` @@ -1405,7 +1402,7 @@ must specify for any option using that action. ``print_version()`` method of OptionParser. Generally only relevant if the ``version`` argument is supplied to the OptionParser constructor. As with :attr:`~Option.help` options, you will rarely create ``version`` options, - since :mod:`optparse` automatically adds them when needed. + since :mod:`!optparse` automatically adds them when needed. .. _optparse-standard-option-types: @@ -1413,7 +1410,7 @@ must specify for any option using that action. Standard option types ^^^^^^^^^^^^^^^^^^^^^ -:mod:`optparse` has five built-in option types: ``"string"``, ``"int"``, +:mod:`!optparse` has five built-in option types: ``"string"``, ``"int"``, ``"choice"``, ``"float"`` and ``"complex"``. If you need to add new option types, see section :ref:`optparse-extending-optparse`. @@ -1432,7 +1429,7 @@ Integer arguments (type ``"int"``) are parsed as follows: The conversion is done by calling :func:`int` with the appropriate base (2, 8, -10, or 16). If this fails, so will :mod:`optparse`, although with a more useful +10, or 16). If this fails, so will :mod:`!optparse`, although with a more useful error message. ``"float"`` and ``"complex"`` option arguments are converted directly with @@ -1471,7 +1468,7 @@ The whole point of creating and populating an OptionParser is to call its ``options`` the same object that was passed in as *values*, or the ``optparse.Values`` - instance created by :mod:`optparse` + instance created by :mod:`!optparse` ``args`` the leftover positional arguments after all options have been processed @@ -1499,7 +1496,7 @@ provides several methods to help you out: .. method:: OptionParser.disable_interspersed_args() Set parsing to stop on the first non-option. For example, if ``-a`` and - ``-b`` are both simple options that take no arguments, :mod:`optparse` + ``-b`` are both simple options that take no arguments, :mod:`!optparse` normally accepts this syntax:: prog -a arg1 -b arg2 @@ -1554,7 +1551,7 @@ strings:: (This is particularly true if you've defined your own OptionParser subclass with some standard options.) -Every time you add an option, :mod:`optparse` checks for conflicts with existing +Every time you add an option, :mod:`!optparse` checks for conflicts with existing options. If it finds any, it invokes the current conflict-handling mechanism. You can set the conflict-handling mechanism either in the constructor:: @@ -1581,7 +1578,7 @@ intelligently and add conflicting options to it:: parser.add_option("-n", "--dry-run", ..., help="do no harm") parser.add_option("-n", "--noisy", ..., help="be noisy") -At this point, :mod:`optparse` detects that a previously added option is already +At this point, :mod:`!optparse` detects that a previously added option is already using the ``-n`` option string. Since ``conflict_handler`` is ``"resolve"``, it resolves the situation by removing ``-n`` from the earlier option's list of option strings. Now ``--dry-run`` is the only way for the user to activate @@ -1594,14 +1591,14 @@ that option. If the user asks for help, the help message will reflect that:: It's possible to whittle away the option strings for a previously added option until there are none left, and the user has no way of invoking that option from -the command-line. In that case, :mod:`optparse` removes that option completely, +the command-line. In that case, :mod:`!optparse` removes that option completely, so it doesn't show up in help text or anywhere else. Carrying on with our existing OptionParser:: parser.add_option("--dry-run", ..., help="new dry-run option") At this point, the original ``-n``/``--dry-run`` option is no longer -accessible, so :mod:`optparse` removes it, leaving this help text:: +accessible, so :mod:`!optparse` removes it, leaving this help text:: Options: ... @@ -1676,9 +1673,9 @@ OptionParser supports several other public methods: Option Callbacks ---------------- -When :mod:`optparse`'s built-in actions and types aren't quite enough for your -needs, you have two choices: extend :mod:`optparse` or define a callback option. -Extending :mod:`optparse` is more general, but overkill for a lot of simple +When :mod:`!optparse`'s built-in actions and types aren't quite enough for your +needs, you have two choices: extend :mod:`!optparse` or define a callback option. +Extending :mod:`!optparse` is more general, but overkill for a lot of simple cases. Quite often a simple callback is all you need. There are two steps to defining a callback option: @@ -1702,14 +1699,14 @@ only option attribute you must specify is ``callback``, the function to call:: ``callback`` is a function (or other callable object), so you must have already defined ``my_callback()`` when you create this callback option. In this simple -case, :mod:`optparse` doesn't even know if ``-c`` takes any arguments, +case, :mod:`!optparse` doesn't even know if ``-c`` takes any arguments, which usually means that the option takes no arguments---the mere presence of ``-c`` on the command-line is all it needs to know. In some circumstances, though, you might want your callback to consume an arbitrary number of command-line arguments. This is where writing callbacks gets tricky; it's covered later in this section. -:mod:`optparse` always passes four particular arguments to your callback, and it +:mod:`!optparse` always passes four particular arguments to your callback, and it will only pass additional arguments if you specify them via :attr:`~Option.callback_args` and :attr:`~Option.callback_kwargs`. Thus, the minimal callback function signature is:: @@ -1723,12 +1720,12 @@ callback option: :attr:`~Option.type` has its usual meaning: as with the ``"store"`` or ``"append"`` actions, it - instructs :mod:`optparse` to consume one argument and convert it to + instructs :mod:`!optparse` to consume one argument and convert it to :attr:`~Option.type`. Rather than storing the converted value(s) anywhere, - though, :mod:`optparse` passes it to your callback function. + though, :mod:`!optparse` passes it to your callback function. :attr:`~Option.nargs` - also has its usual meaning: if it is supplied and > 1, :mod:`optparse` will + also has its usual meaning: if it is supplied and > 1, :mod:`!optparse` will consume :attr:`~Option.nargs` arguments, each of which must be convertible to :attr:`~Option.type`. It then passes a tuple of converted values to your callback. @@ -1762,7 +1759,7 @@ where ``"--foobar"``.) ``value`` - is the argument to this option seen on the command-line. :mod:`optparse` will + is the argument to this option seen on the command-line. :mod:`!optparse` will only expect an argument if :attr:`~Option.type` is set; the type of ``value`` will be the type implied by the option's type. If :attr:`~Option.type` for this option is ``None`` (no argument expected), then ``value`` will be ``None``. If :attr:`~Option.nargs` @@ -1787,7 +1784,7 @@ where ``parser.values`` the object where option values are by default stored (an instance of optparse.OptionValues). This lets callbacks use the same mechanism as the - rest of :mod:`optparse` for storing option values; you don't need to mess + rest of :mod:`!optparse` for storing option values; you don't need to mess around with globals or closures. You can also access or modify the value(s) of any options already encountered on the command-line. @@ -1806,7 +1803,7 @@ Raising errors in a callback ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The callback function should raise :exc:`OptionValueError` if there are any -problems with the option or its argument(s). :mod:`optparse` catches this and +problems with the option or its argument(s). :mod:`!optparse` catches this and terminates the program, printing the error message you supply to stderr. Your message should be clear, concise, accurate, and mention the option at fault. Otherwise, the user will have a hard time figuring out what they did wrong. @@ -1906,7 +1903,7 @@ Here's an example that just emulates the standard ``"store"`` action:: action="callback", callback=store_value, type="int", nargs=3, dest="foo") -Note that :mod:`optparse` takes care of consuming 3 arguments and converting +Note that :mod:`!optparse` takes care of consuming 3 arguments and converting them to integers for you; all you have to do is store them. (Or whatever; obviously you don't need a callback for this example.) @@ -1917,9 +1914,9 @@ Callback example 6: variable arguments ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Things get hairy when you want an option to take a variable number of arguments. -For this case, you must write a callback, as :mod:`optparse` doesn't provide any +For this case, you must write a callback, as :mod:`!optparse` doesn't provide any built-in capabilities for it. And you have to deal with certain intricacies of -conventional Unix command-line parsing that :mod:`optparse` normally handles for +conventional Unix command-line parsing that :mod:`!optparse` normally handles for you. In particular, callbacks should implement the conventional rules for bare ``--`` and ``-`` arguments: @@ -1934,7 +1931,7 @@ you. In particular, callbacks should implement the conventional rules for bare If you want an option that takes a variable number of arguments, there are several subtle, tricky issues to worry about. The exact implementation you choose will be based on which trade-offs you're willing to make for your -application (which is why :mod:`optparse` doesn't support this sort of thing +application (which is why :mod:`!optparse` doesn't support this sort of thing directly). Nevertheless, here's a stab at a callback for an option with variable @@ -1970,10 +1967,10 @@ arguments:: .. _optparse-extending-optparse: -Extending :mod:`optparse` -------------------------- +Extending :mod:`!optparse` +-------------------------- -Since the two major controlling factors in how :mod:`optparse` interprets +Since the two major controlling factors in how :mod:`!optparse` interprets command-line options are the action and type of each option, the most likely direction of extension is to add new actions and new types. @@ -1983,9 +1980,9 @@ direction of extension is to add new actions and new types. Adding new types ^^^^^^^^^^^^^^^^ -To add new types, you need to define your own subclass of :mod:`optparse`'s +To add new types, you need to define your own subclass of :mod:`!optparse`'s :class:`Option` class. This class has a couple of attributes that define -:mod:`optparse`'s types: :attr:`~Option.TYPES` and :attr:`~Option.TYPE_CHECKER`. +:mod:`!optparse`'s types: :attr:`~Option.TYPES` and :attr:`~Option.TYPE_CHECKER`. .. attribute:: Option.TYPES @@ -2015,7 +2012,7 @@ To add new types, you need to define your own subclass of :mod:`optparse`'s Here's a silly example that demonstrates adding a ``"complex"`` option type to parse Python-style complex numbers on the command line. (This is even sillier -than it used to be, because :mod:`optparse` 1.3 added built-in support for +than it used to be, because :mod:`!optparse` 1.3 added built-in support for complex numbers, but never mind.) First, the necessary imports:: @@ -2041,12 +2038,12 @@ Finally, the Option subclass:: TYPE_CHECKER["complex"] = check_complex (If we didn't make a :func:`copy` of :attr:`Option.TYPE_CHECKER`, we would end -up modifying the :attr:`~Option.TYPE_CHECKER` attribute of :mod:`optparse`'s +up modifying the :attr:`~Option.TYPE_CHECKER` attribute of :mod:`!optparse`'s Option class. This being Python, nothing stops you from doing that except good manners and common sense.) That's it! Now you can write a script that uses the new option type just like -any other :mod:`optparse`\ -based script, except you have to instruct your +any other :mod:`!optparse`\ -based script, except you have to instruct your OptionParser to use MyOption instead of Option:: parser = OptionParser(option_class=MyOption) @@ -2066,10 +2063,10 @@ Adding new actions ^^^^^^^^^^^^^^^^^^ Adding new actions is a bit trickier, because you have to understand that -:mod:`optparse` has a couple of classifications for actions: +:mod:`!optparse` has a couple of classifications for actions: "store" actions - actions that result in :mod:`optparse` storing a value to an attribute of the + actions that result in :mod:`!optparse` storing a value to an attribute of the current OptionValues instance; these options require a :attr:`~Option.dest` attribute to be supplied to the Option constructor. @@ -2101,7 +2098,7 @@ of the following class attributes of Option (all are lists of strings): .. attribute:: Option.ALWAYS_TYPED_ACTIONS Actions that always take a type (i.e. whose options always take a value) are - additionally listed here. The only effect of this is that :mod:`optparse` + additionally listed here. The only effect of this is that :mod:`!optparse` assigns the default type, ``"string"``, to options with no explicit type whose action is listed in :attr:`ALWAYS_TYPED_ACTIONS`. @@ -2144,12 +2141,12 @@ Features of note: somewhere, so it goes in both :attr:`~Option.STORE_ACTIONS` and :attr:`~Option.TYPED_ACTIONS`. -* to ensure that :mod:`optparse` assigns the default type of ``"string"`` to +* to ensure that :mod:`!optparse` assigns the default type of ``"string"`` to ``"extend"`` actions, we put the ``"extend"`` action in :attr:`~Option.ALWAYS_TYPED_ACTIONS` as well. * :meth:`MyOption.take_action` implements just this one new action, and passes - control back to :meth:`Option.take_action` for the standard :mod:`optparse` + control back to :meth:`Option.take_action` for the standard :mod:`!optparse` actions. * ``values`` is an instance of the optparse_parser.Values class, which provides diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index d27023cde4cfe9f..808187061733be0 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -36,14 +36,14 @@ the :mod:`glob` module.) Since different operating systems have different path name conventions, there are several versions of this module in the standard library. The - :mod:`os.path` module is always the path module suitable for the operating + :mod:`!os.path` module is always the path module suitable for the operating system Python is running on, and therefore usable for local paths. However, you can also import and use the individual modules if you want to manipulate a path that is *always* in one of the different formats. They all have the same interface: - * :mod:`posixpath` for UNIX-style paths - * :mod:`ntpath` for Windows paths + * :mod:`!posixpath` for UNIX-style paths + * :mod:`!ntpath` for Windows paths .. versionchanged:: 3.8 @@ -57,8 +57,9 @@ the :mod:`glob` module.) .. function:: abspath(path) Return a normalized absolutized version of the pathname *path*. On most - platforms, this is equivalent to calling the function :func:`normpath` as - follows: ``normpath(join(os.getcwd(), path))``. + platforms, this is equivalent to calling ``normpath(join(os.getcwd(), path))``. + + .. seealso:: :func:`os.path.join` and :func:`os.path.normpath`. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -96,15 +97,17 @@ the :mod:`glob` module.) .. function:: commonprefix(list, /) - Return the longest path prefix (taken character-by-character) that is a - prefix of all paths in *list*. If *list* is empty, return the empty string + Return the longest string prefix (taken character-by-character) that is a + prefix of all strings in *list*. If *list* is empty, return the empty string (``''``). - .. note:: + .. warning:: This function may return invalid paths because it works a - character at a time. To obtain a valid path, see - :func:`commonpath`. + character at a time. + If you need a **common path prefix**, then the algorithm + implemented in this function is not secure. Use + :func:`commonpath` for finding a common path prefix. :: @@ -117,6 +120,14 @@ the :mod:`glob` module.) .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. deprecated:: 3.15 + Deprecated in favor of :func:`os.path.commonpath` for path prefixes. + The :func:`os.path.commonprefix` function is being deprecated due to + having a misleading name and module. The function is not safe to use for + path prefixes despite being included in a module about path manipulation, + meaning it is easy to accidentally introduce path traversal + vulnerabilities into Python programs by using this function. + .. function:: dirname(path, /) @@ -243,6 +254,8 @@ the :mod:`glob` module.) begins with a slash, on Windows that it begins with two (back)slashes, or a drive letter, colon, and (back)slash together. + .. seealso:: :func:`abspath` + .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -357,14 +370,28 @@ the :mod:`glob` module.) concatenation of *path* and all members of *\*paths*, with exactly one directory separator following each non-empty part, except the last. That is, the result will only end in a separator if the last part is either empty or - ends in a separator. If a segment is an absolute path (which on Windows - requires both a drive and a root), then all previous segments are ignored and - joining continues from the absolute path segment. + ends in a separator. + + If a segment is an absolute path (which on Windows requires both a drive and + a root), then all previous segments are ignored and joining continues from the + absolute path segment. On Linux, for example:: + + >>> os.path.join('/home/foo', 'bar') + '/home/foo/bar' + >>> os.path.join('/home/foo', '/home/bar') + '/home/bar' On Windows, the drive is not reset when a rooted path segment (e.g., ``r'\foo'``) is encountered. If a segment is on a different drive or is an - absolute path, all previous segments are ignored and the drive is reset. Note - that since there is a current directory for each drive, + absolute path, all previous segments are ignored and the drive is reset. For + example:: + + >>> os.path.join('c:\\', 'foo') + 'c:\\foo' + >>> os.path.join('c:\\foo', 'd:\\bar') + 'd:\\bar' + + Note that since there is a current directory for each drive, ``os.path.join("c:", "foo")`` represents a path relative to the current directory on drive :file:`C:` (:file:`c:foo`), not :file:`c:\\foo`. @@ -424,6 +451,8 @@ the :mod:`glob` module.) re-raised. In particular, :exc:`FileNotFoundError` is raised if *path* does not exist, or another :exc:`OSError` if it is otherwise inaccessible. + If *strict* is :data:`ALL_BUT_LAST`, the last component of the path + is allowed to be missing, but all other errors are raised. If *strict* is :py:data:`os.path.ALLOW_MISSING`, errors other than :exc:`FileNotFoundError` are re-raised (as with ``strict=True``). @@ -447,15 +476,22 @@ the :mod:`glob` module.) .. versionchanged:: 3.10 The *strict* parameter was added. - .. versionchanged:: next - The :py:data:`~os.path.ALLOW_MISSING` value for the *strict* parameter - was added. + .. versionchanged:: 3.15 + The :data:`ALL_BUT_LAST` and :data:`ALLOW_MISSING` values for + the *strict* parameter was added. + +.. data:: ALL_BUT_LAST + + Special value used for the *strict* argument in :func:`realpath`. + + .. versionadded:: 3.15 .. data:: ALLOW_MISSING Special value used for the *strict* argument in :func:`realpath`. - .. versionadded:: next + .. versionadded:: 3.15 + .. function:: relpath(path, start=os.curdir) @@ -518,8 +554,8 @@ the :mod:`glob` module.) *path* is empty, both *head* and *tail* are empty. Trailing slashes are stripped from *head* unless it is the root (one or more slashes only). In all cases, ``join(head, tail)`` returns a path to the same location as *path* - (but the strings may differ). Also see the functions :func:`dirname` and - :func:`basename`. + (but the strings may differ). Also see the functions :func:`join`, + :func:`dirname` and :func:`basename`. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 45ec6c7a51b7b02..7b043f257ca0b57 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -25,7 +25,7 @@ Notes on the availability of these functions: with the POSIX interface). * Extensions peculiar to a particular operating system are also available - through the :mod:`os` module, but using them is of course a threat to + through the :mod:`!os` module, but using them is of course a threat to portability. * All functions accepting path or file names accept both bytes and string @@ -34,7 +34,7 @@ Notes on the availability of these functions: * On VxWorks, os.popen, os.fork, os.execv and os.spawn*p* are not supported. -* On WebAssembly platforms, Android and iOS, large parts of the :mod:`os` module are +* On WebAssembly platforms, Android and iOS, large parts of the :mod:`!os` module are not available or behave differently. APIs related to processes (e.g. :func:`~os.fork`, :func:`~os.execve`) and resources (e.g. :func:`~os.nice`) are not available. Others like :func:`~os.getuid` and :func:`~os.getpid` are @@ -108,10 +108,10 @@ Python UTF-8 Mode .. versionadded:: 3.7 See :pep:`540` for more details. -.. versionchanged:: next +.. versionchanged:: 3.15 Python UTF-8 mode is now enabled by default (:pep:`686`). - It may be disabled with by setting :envvar:`PYTHONUTF8=0 ` as + It may be disabled by setting :envvar:`PYTHONUTF8=0 ` as an environment variable or by using the :option:`-X utf8=0 <-X>` command line option. The Python UTF-8 Mode ignores the :term:`locale encoding` and forces the usage @@ -185,7 +185,7 @@ process and user. of your home directory (on some platforms), and is equivalent to ``getenv("HOME")`` in C. - This mapping is captured the first time the :mod:`os` module is imported, + This mapping is captured the first time the :mod:`!os` module is imported, typically during Python startup as part of processing :file:`site.py`. Changes to the environment made after this time are not reflected in :data:`os.environ`, except for changes made by modifying :data:`os.environ` directly. @@ -216,8 +216,16 @@ process and user. You can delete items in this mapping to unset environment variables. :func:`unsetenv` will be called automatically when an item is deleted from - :data:`os.environ`, and when one of the :meth:`pop` or :meth:`clear` methods is - called. + :data:`os.environ`, and when one of the :meth:`~dict.pop` or + :meth:`~dict.clear` methods is called. + + If the :manpage:`clearenv(3)` function is available, the :meth:`~dict.clear` method + uses it and emits a single ``os._clearenv`` audit event. Otherwise, it emits + an ``os.unsetenv`` event on each deleted variable. + + .. audit-event:: os.unsetenv key os.unsetenv + + .. audit-event:: os._clearenv "" os._clearenv .. seealso:: @@ -226,6 +234,10 @@ process and user. .. versionchanged:: 3.9 Updated to support :pep:`584`'s merge (``|``) and update (``|=``) operators. + .. versionchanged:: 3.15 + The :meth:`~dict.clear` method can now emit an ``os._clearenv`` audit + event. + .. data:: environb @@ -254,7 +266,7 @@ process and user. .. warning:: This function is not thread-safe. Calling it while the environment is - being modified in an other thread is an undefined behavior. Reading from + being modified in another thread is an undefined behavior. Reading from :data:`os.environ` or :data:`os.environb`, or calling :func:`os.getenv` while reloading, may return an empty result. @@ -430,8 +442,8 @@ process and user. associated with the effective user id of the process; the group access list may change over the lifetime of the process, it is not affected by calls to :func:`setgroups`, and its length is not limited to 16. The - deployment target value, :const:`MACOSX_DEPLOYMENT_TARGET`, can be - obtained with :func:`sysconfig.get_config_var`. + deployment target value can be obtained with + :func:`sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') `. .. function:: getlogin() @@ -529,7 +541,7 @@ process and user. Return a tuple (ruid, euid, suid) denoting the current process's real, effective, and saved user ids. - .. availability:: Unix, not WASI. + .. availability:: Unix, not WASI, not macOS, not iOS. .. versionadded:: 3.2 @@ -539,7 +551,7 @@ process and user. Return a tuple (rgid, egid, sgid) denoting the current process's real, effective, and saved group ids. - .. availability:: Unix, not WASI. + .. availability:: Unix, not WASI, not macOS, not iOS. .. versionadded:: 3.2 @@ -558,7 +570,7 @@ process and user. .. function:: initgroups(username, gid, /) - Call the system initgroups() to initialize the group access list with all of + Call the system ``initgroups()`` to initialize the group access list with all of the groups of which the specified username is a member, plus the specified group id. @@ -713,7 +725,7 @@ process and user. Set the current process's real, effective, and saved group ids. - .. availability:: Unix, not WASI, not Android. + .. availability:: Unix, not WASI, not Android, not macOS, not iOS. .. versionadded:: 3.2 @@ -722,7 +734,7 @@ process and user. Set the current process's real, effective, and saved user ids. - .. availability:: Unix, not WASI, not Android. + .. availability:: Unix, not WASI, not Android, not macOS, not iOS. .. versionadded:: 3.2 @@ -788,29 +800,19 @@ process and user. single: gethostbyaddr() (in module socket) Returns information identifying the current operating system. - The return value is an object with five attributes: - - * :attr:`sysname` - operating system name - * :attr:`nodename` - name of machine on network (implementation-defined) - * :attr:`release` - operating system release - * :attr:`version` - operating system version - * :attr:`machine` - hardware identifier - - For backwards compatibility, this object is also iterable, behaving - like a five-tuple containing :attr:`sysname`, :attr:`nodename`, - :attr:`release`, :attr:`version`, and :attr:`machine` - in that order. - - Some systems truncate :attr:`nodename` to 8 characters or to the - leading component; a better way to get the hostname is - :func:`socket.gethostname` or even - ``socket.gethostbyaddr(socket.gethostname())``. + The return value is a :class:`uname_result`. On macOS, iOS and Android, this returns the *kernel* name and version (i.e., ``'Darwin'`` on macOS and iOS; ``'Linux'`` on Android). :func:`platform.uname` can be used to get the user-facing operating system name and version on iOS and Android. + .. seealso:: + :data:`sys.platform` which has finer granularity. + + The :mod:`platform` module provides detailed checks for the + system's identity. + .. availability:: Unix. .. versionchanged:: 3.3 @@ -818,6 +820,41 @@ process and user. with named attributes. +.. class:: uname_result + + Name and information about the system returned by :func:`os.uname`. + These attributes correspond to the members described in :manpage:`uname(2)`. + + For backwards compatibility, this object is also iterable, behaving + like a five-tuple containing :attr:`~uname_result.sysname`, + :attr:`~uname_result.nodename`, :attr:`~uname_result.release`, + :attr:`~uname_result.version`, and :attr:`~uname_result.machine` + in that order. + + .. attribute:: sysname + + Operating system name. + + .. attribute:: nodename + + Name of machine on network. Some systems truncate + :attr:`~uname_result.nodename` to 8 characters or to the leading + component; a better way to get the hostname is :func:`socket.gethostname` + or even ``socket.gethostbyaddr(socket.gethostname())``. + + .. attribute:: release + + Operating system release. + + .. attribute:: version + + Operating system version. + + .. attribute:: machine + + Hardware identifier. + + .. function:: unsetenv(key, /) .. index:: single: environment variables; deleting @@ -1059,10 +1096,7 @@ as internal buffering of data. Force write of file with filedescriptor *fd* to disk. Does not force update of metadata. - .. availability:: Unix. - - .. note:: - This function is not available on MacOS. + .. availability:: Unix, not macOS, not iOS. .. function:: fpathconf(fd, name, /) @@ -1100,8 +1134,8 @@ as internal buffering of data. .. function:: fstatvfs(fd, /) Return information about the filesystem containing the file associated with - file descriptor *fd*, like :func:`statvfs`. As of Python 3.3, this is - equivalent to ``os.statvfs(fd)``. + file descriptor *fd* in a :class:`statvfs_result`, like :func:`statvfs`. + As of Python 3.3, this is equivalent to ``os.statvfs(fd)``. .. availability:: Unix. @@ -1279,7 +1313,7 @@ as internal buffering of data. For a description of the flag and mode values, see the C run-time documentation; flag constants (like :const:`O_RDONLY` and :const:`O_WRONLY`) are defined in - the :mod:`os` module. In particular, on Windows adding + the :mod:`!os` module. In particular, on Windows adding :const:`O_BINARY` is needed to open files in binary mode. This function can support :ref:`paths relative to directory descriptors @@ -1294,8 +1328,8 @@ as internal buffering of data. This function is intended for low-level I/O. For normal usage, use the built-in function :func:`open`, which returns a :term:`file object` with - :meth:`~file.read` and :meth:`~file.write` methods (and many more). To - wrap a file descriptor in a file object, use :func:`fdopen`. + :meth:`~io.BufferedIOBase.read` and :meth:`~io.BufferedIOBase.write` methods. + To wrap a file descriptor in a file object, use :func:`fdopen`. .. versionchanged:: 3.3 Added the *dir_fd* parameter. @@ -1414,7 +1448,7 @@ or `the MSDN `_ on Windo Return a pair of file descriptors ``(r, w)`` usable for reading and writing, respectively. - .. availability:: Unix, not WASI. + .. availability:: Unix, not WASI, not macOS, not iOS. .. versionadded:: 3.3 @@ -1424,7 +1458,7 @@ or `the MSDN `_ on Windo Ensures that enough disk space is allocated for the file specified by *fd* starting from *offset* and continuing for *len* bytes. - .. availability:: Unix. + .. availability:: Unix, not macOS, not iOS. .. versionadded:: 3.3 @@ -1439,7 +1473,7 @@ or `the MSDN `_ on Windo :data:`POSIX_FADV_RANDOM`, :data:`POSIX_FADV_NOREUSE`, :data:`POSIX_FADV_WILLNEED` or :data:`POSIX_FADV_DONTNEED`. - .. availability:: Unix. + .. availability:: Unix, not macOS, not iOS. .. versionadded:: 3.3 @@ -1501,6 +1535,7 @@ or `the MSDN `_ on Windo - :data:`RWF_HIPRI` - :data:`RWF_NOWAIT` + - :data:`RWF_DONTCACHE` Return the total number of bytes actually read which can be less than the total capacity of all the objects. @@ -1546,6 +1581,24 @@ or `the MSDN `_ on Windo .. versionadded:: 3.7 +.. data:: RWF_DONTCACHE + + Use uncached buffered IO. + + .. availability:: Linux >= 6.14 + + .. versionadded:: 3.15 + + +.. data:: RWF_ATOMIC + + Write data atomically. Requires alignment to the device's atomic write unit. + + .. availability:: Linux >= 6.11 + + .. versionadded:: 3.15 + + .. function:: ptsname(fd, /) Return the name of the slave pseudo-terminal device associated with the @@ -1587,6 +1640,8 @@ or `the MSDN `_ on Windo - :data:`RWF_DSYNC` - :data:`RWF_SYNC` - :data:`RWF_APPEND` + - :data:`RWF_DONTCACHE` + - :data:`RWF_ATOMIC` Return the total number of bytes actually written. @@ -1649,7 +1704,7 @@ or `the MSDN `_ on Windo descriptor as returned by :func:`os.open` or :func:`pipe`. To read a "file object" returned by the built-in function :func:`open` or by :func:`popen` or :func:`fdopen`, or :data:`sys.stdin`, use its - :meth:`~file.read` or :meth:`~file.readline` methods. + :meth:`~io.TextIOBase.read` or :meth:`~io.IOBase.readline` methods. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an @@ -1884,7 +1939,7 @@ or `the MSDN `_ on Windo descriptor as returned by :func:`os.open` or :func:`pipe`. To write a "file object" returned by the built-in function :func:`open` or by :func:`popen` or :func:`fdopen`, or :data:`sys.stdout` or :data:`sys.stderr`, use its - :meth:`~file.write` method. + :meth:`~io.TextIOBase.write` method. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an @@ -1958,7 +2013,8 @@ can be inherited by child processes. Since Python 3.4, file descriptors created by Python are non-inheritable by default. On UNIX, non-inheritable file descriptors are closed in child processes at the -execution of a new program, other file descriptors are inherited. +execution of a new program, other file descriptors are inherited. Note that +non-inheritable file descriptors are still *inherited* by child processes on :func:`os.fork`. On Windows, non-inheritable handles and file descriptors are closed in child processes, except for standard streams (file descriptors 0, 1 and 2: stdin, stdout @@ -2002,12 +2058,12 @@ features: .. _path_fd: * **specifying a file descriptor:** - Normally the *path* argument provided to functions in the :mod:`os` module + Normally the *path* argument provided to functions in the :mod:`!os` module must be a string specifying a file path. However, some functions now alternatively accept an open file descriptor for their *path* argument. The function will then operate on the file referred to by the descriptor. - (For POSIX systems, Python will call the variant of the function prefixed - with ``f`` (e.g. call ``fchdir`` instead of ``chdir``).) + For POSIX systems, Python will call the variant of the function prefixed + with ``f`` (e.g. call ``fchdir`` instead of ``chdir``). You can check whether or not *path* can be specified as a file descriptor for a particular function on your platform using :data:`os.supports_fd`. @@ -2022,7 +2078,7 @@ features: * **paths relative to directory descriptors:** If *dir_fd* is not ``None``, it should be a file descriptor referring to a directory, and the path to operate on should be relative; path will then be relative to that directory. If the - path is absolute, *dir_fd* is ignored. (For POSIX systems, Python will call + path is absolute, *dir_fd* is ignored. For POSIX systems, Python will call the variant of the function with an ``at`` suffix and possibly prefixed with ``f`` (e.g. call ``faccessat`` instead of ``access``). @@ -2035,8 +2091,8 @@ features: * **not following symlinks:** If *follow_symlinks* is ``False``, and the last element of the path to operate on is a symbolic link, the function will operate on the symbolic link itself rather than the file - pointed to by the link. (For POSIX systems, Python will call the ``l...`` - variant of the function.) + pointed to by the link. For POSIX systems, Python will call the ``l...`` + variant of the function. You can check whether or not *follow_symlinks* is supported for a particular function on your platform using :data:`os.supports_follow_symlinks`. @@ -2387,6 +2443,10 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.15 + ``os.listdir(-1)`` now fails with ``OSError(errno.EBADF)`` rather than + listing the current directory. + .. function:: listdrives() @@ -2523,7 +2583,8 @@ features: Windows now handles a *mode* of ``0o700``. -.. function:: makedirs(name, mode=0o777, exist_ok=False) +.. function:: makedirs(name, mode=0o777, exist_ok=False, *, \ + parent_mode=None) .. index:: single: directory; creating @@ -2541,6 +2602,12 @@ features: If *exist_ok* is ``False`` (the default), a :exc:`FileExistsError` is raised if the target directory already exists. + If *parent_mode* is not ``None``, it is used as the mode for any + newly-created, intermediate-level directories. Like *mode*, it is + combined with the process's umask value; see :ref:`the mkdir() + description `. Otherwise, intermediate directories are + created with the default mode, which is also subject to the umask. + .. note:: :func:`makedirs` will become confused if the path elements to create @@ -2567,6 +2634,11 @@ features: The *mode* argument no longer affects the file permission bits of newly created intermediate-level directories. + .. versionadded:: 3.15 + The *parent_mode* parameter. To match the behavior from Python 3.6 and + earlier (where *mode* was applied to all created directories), pass + ``parent_mode=mode``. + .. function:: mkfifo(path, mode=0o666, *, dir_fd=None) @@ -2595,10 +2667,10 @@ features: Create a filesystem node (file, device special file or named pipe) named *path*. *mode* specifies both the permissions to use and the type of node - to be created, being combined (bitwise OR) with one of ``stat.S_IFREG``, - ``stat.S_IFCHR``, ``stat.S_IFBLK``, and ``stat.S_IFIFO`` (those constants are - available in :mod:`stat`). For ``stat.S_IFCHR`` and ``stat.S_IFBLK``, - *device* defines the newly created device special file (probably using + to be created, being combined (bitwise OR) with one of :const:`stat.S_IFREG`, + :const:`stat.S_IFCHR`, :const:`stat.S_IFBLK`, and :const:`stat.S_IFIFO`. + For :const:`stat.S_IFCHR` and :const:`stat.S_IFBLK`, *device* defines the + newly created device special file (probably using :func:`os.makedev`), otherwise it is ignored. This function can also support :ref:`paths relative to directory descriptors @@ -2616,13 +2688,13 @@ features: .. function:: major(device, /) Extract the device major number from a raw device number (usually the - :attr:`st_dev` or :attr:`st_rdev` field from :c:struct:`stat`). + :attr:`~stat_result.st_dev` or :attr:`~stat_result.st_rdev` field from :c:struct:`stat`). .. function:: minor(device, /) Extract the device minor number from a raw device number (usually the - :attr:`st_dev` or :attr:`st_rdev` field from :c:struct:`stat`). + :attr:`~stat_result.st_dev` or :attr:`~stat_result.st_rdev` field from :c:struct:`stat`). .. function:: makedev(major, minor, /) @@ -2630,6 +2702,13 @@ features: Compose a raw device number from the major and minor device numbers. +.. data:: NODEV + + Non-existent device. + + .. versionadded:: 3.15 + + .. function:: pathconf(path, name) Return system configuration information relevant to a named file. *name* @@ -2910,6 +2989,10 @@ features: .. versionchanged:: 3.7 Added support for :ref:`file descriptors ` on Unix. + .. versionchanged:: 3.15 + ``os.scandir(-1)`` now fails with ``OSError(errno.EBADF)`` rather than + listing the current directory. + .. class:: DirEntry @@ -2933,6 +3016,9 @@ features: To be directly usable as a :term:`path-like object`, ``os.DirEntry`` implements the :class:`PathLike` interface. + :class:`!DirEntry` objects are :ref:`generic ` over the type of the + path (:class:`str` or :class:`bytes`). + Attributes and methods on a ``os.DirEntry`` instance are as follows: .. attribute:: name @@ -3346,8 +3432,8 @@ features: .. versionchanged:: 3.8 On Windows, the :attr:`st_mode` member now identifies special - files as :const:`S_IFCHR`, :const:`S_IFIFO` or :const:`S_IFBLK` - as appropriate. + files as :const:`~stat.S_IFCHR`, :const:`~stat.S_IFIFO` or + :const:`~stat.S_IFBLK` as appropriate. .. versionchanged:: 3.12 On Windows, :attr:`st_ctime` is deprecated. Eventually, it will @@ -3365,55 +3451,534 @@ features: Added the :attr:`st_birthtime` member on Windows. +.. function:: statx(path, mask, *, flags=0, dir_fd=None, follow_symlinks=True) + + Get the status of a file or file descriptor by performing a :c:func:`!statx` + system call on the given path. + + *path* is a :term:`path-like object` or an open file descriptor. *mask* is a + combination of the module-level :const:`STATX_* ` constants + specifying the information to retrieve. *flags* is a combination of the + module-level :const:`AT_STATX_* ` constants and/or + :const:`AT_NO_AUTOMOUNT`. Returns a :class:`statx_result` object whose + :attr:`~os.statx_result.stx_mask` attribute specifies the information + actually retrieved (which may differ from *mask*). + + This function supports :ref:`specifying a file descriptor `, + :ref:`paths relative to directory descriptors `, and + :ref:`not following symlinks `. + + .. seealso:: The :manpage:`statx(2)` man page. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: 3.15 + + +.. class:: statx_result + + Information about a file returned by :func:`os.statx`. + + :class:`!statx_result` has the following attributes: + + .. attribute:: stx_atime + + Time of most recent access expressed in seconds. + + Equal to ``None`` if :data:`STATX_ATIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_atime_ns + + Time of most recent access expressed in nanoseconds as an integer. + + Equal to ``None`` if :data:`STATX_ATIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_atomic_write_segments_max + + Maximum iovecs for direct I/O with torn-write protection. + + Equal to ``None`` if :data:`STATX_WRITE_ATOMIC` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.11. + + .. attribute:: stx_atomic_write_unit_max + + Maximum size for direct I/O with torn-write protection. + + Equal to ``None`` if :data:`STATX_WRITE_ATOMIC` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.11. + + .. attribute:: stx_atomic_write_unit_max_opt + + Maximum optimized size for direct I/O with torn-write protection. + + Equal to ``None`` if :data:`STATX_WRITE_ATOMIC` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.16. + + .. attribute:: stx_atomic_write_unit_min + + Minimum size for direct I/O with torn-write protection. + + Equal to ``None`` if :data:`STATX_WRITE_ATOMIC` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.11. + + .. attribute:: stx_attributes + + Bitmask of :const:`STATX_ATTR_* ` constants + specifying the attributes of this file. + + .. attribute:: stx_attributes_mask + + A mask indicating which bits in :attr:`stx_attributes` are supported by + the VFS and the filesystem. + + .. attribute:: stx_blksize + + "Preferred" blocksize for efficient file system I/O. Writing to a file in + smaller chunks may cause an inefficient read-modify-rewrite. + + .. attribute:: stx_blocks + + Number of 512-byte blocks allocated for file. + This may be smaller than :attr:`stx_size`/512 when the file has holes. + + Equal to ``None`` if :data:`STATX_BLOCKS` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_btime + + Time of file creation expressed in seconds. + + Equal to ``None`` if :data:`STATX_BTIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_btime_ns + + Time of file creation expressed in nanoseconds as an integer. + + Equal to ``None`` if :data:`STATX_BTIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_ctime + + Time of most recent metadata change expressed in seconds. + + Equal to ``None`` if :data:`STATX_CTIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_ctime_ns + + Time of most recent metadata change expressed in nanoseconds as an + integer. + + Equal to ``None`` if :data:`STATX_CTIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_dev + + Identifier of the device on which this file resides. + + .. attribute:: stx_dev_major + + Major number of the device on which this file resides. + + .. attribute:: stx_dev_minor + + Minor number of the device on which this file resides. + + .. attribute:: stx_dio_mem_align + + Direct I/O memory buffer alignment requirement. + + Equal to ``None`` if :data:`STATX_DIOALIGN` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.1. + + .. attribute:: stx_dio_offset_align + + Direct I/O file offset alignment requirement. + + Equal to ``None`` if :data:`STATX_DIOALIGN` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.1. + + .. attribute:: stx_dio_read_offset_align + + Direct I/O file offset alignment requirement for reads. + + Equal to ``None`` if :data:`STATX_DIO_READ_ALIGN` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.14. + + .. attribute:: stx_gid + + Group identifier of the file owner. + + Equal to ``None`` if :data:`STATX_GID` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_ino + + Inode number. + + Equal to ``None`` if :data:`STATX_INO` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_mask + + Bitmask of :const:`STATX_* ` constants specifying the + information retrieved, which may differ from what was requested. + + .. attribute:: stx_mnt_id + + Mount identifier. + + Equal to ``None`` if :data:`STATX_MNT_ID` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 5.8. + + .. attribute:: stx_mode + + File mode: file type and file mode bits (permissions). + + Equal to ``None`` if :data:`STATX_TYPE | STATX_MODE ` + is missing from :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_mtime + + Time of most recent content modification expressed in seconds. + + Equal to ``None`` if :data:`STATX_MTIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_mtime_ns + + Time of most recent content modification expressed in nanoseconds as an + integer. + + Equal to ``None`` if :data:`STATX_MTIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_nlink + + Number of hard links. + + Equal to ``None`` if :data:`STATX_NLINK` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_rdev + + Type of device if an inode device. + + .. attribute:: stx_rdev_major + + Major number of the device this file represents. + + .. attribute:: stx_rdev_minor + + Minor number of the device this file represents. + + .. attribute:: stx_size + + Size of the file in bytes, if it is a regular file or a symbolic link. + The size of a symbolic link is the length of the pathname it contains, + without a terminating null byte. + + Equal to ``None`` if :data:`STATX_SIZE` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_subvol + + Subvolume identifier. + + Equal to ``None`` if :data:`STATX_SUBVOL` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.10. + + .. attribute:: stx_uid + + User identifier of the file owner. + + Equal to ``None`` if :data:`STATX_UID` is missing from + :attr:`~statx_result.stx_mask`. + + .. seealso:: The :manpage:`statx(2)` man page. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: 3.15 + + +.. data:: STATX_TYPE + STATX_MODE + STATX_NLINK + STATX_UID + STATX_GID + STATX_ATIME + STATX_MTIME + STATX_CTIME + STATX_INO + STATX_SIZE + STATX_BLOCKS + STATX_BASIC_STATS + STATX_BTIME + STATX_MNT_ID + STATX_DIOALIGN + STATX_MNT_ID_UNIQUE + STATX_SUBVOL + STATX_WRITE_ATOMIC + STATX_DIO_READ_ALIGN + + Bitflags for use in the *mask* parameter to :func:`os.statx`. Some of these + flags may be available even when their corresponding members in + :class:`statx_result` are not available. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: 3.15 + +.. data:: AT_STATX_FORCE_SYNC + + A flag for the :func:`os.statx` function. Requests that the kernel return + up-to-date information even when doing so is expensive (for example, + requiring a round trip to the server for a file on a network filesystem). + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: 3.15 + +.. data:: AT_STATX_DONT_SYNC + + A flag for the :func:`os.statx` function. Requests that the kernel return + cached information if possible. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: 3.15 + +.. data:: AT_STATX_SYNC_AS_STAT + + A flag for the :func:`os.statx` function. This flag is defined as ``0``, so + it has no effect, but it can be used to explicitly indicate neither + :data:`AT_STATX_FORCE_SYNC` nor :data:`AT_STATX_DONT_SYNC` is being passed. + In the absence of the other two flags, the kernel will generally return + information as fresh as :func:`os.stat` would return. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: 3.15 + + +.. data:: AT_NO_AUTOMOUNT + + If the final component of a path is an automount point, operate on the + automount point instead of performing the automount. On Linux, + :func:`os.stat`, :func:`os.fstat` and :func:`os.lstat` always behave this + way. + + .. availability:: Linux. + + .. versionadded:: 3.15 + + .. function:: statvfs(path) - Perform a :c:func:`!statvfs` system call on the given path. The return value is - an object whose attributes describe the filesystem on the given path, and - correspond to the members of the :c:struct:`statvfs` structure, namely: - :attr:`f_bsize`, :attr:`f_frsize`, :attr:`f_blocks`, :attr:`f_bfree`, - :attr:`f_bavail`, :attr:`f_files`, :attr:`f_ffree`, :attr:`f_favail`, - :attr:`f_flag`, :attr:`f_namemax`, :attr:`f_fsid`. - - Two module-level constants are defined for the :attr:`f_flag` attribute's - bit-flags: if :const:`ST_RDONLY` is set, the filesystem is mounted - read-only, and if :const:`ST_NOSUID` is set, the semantics of - setuid/setgid bits are disabled or not supported. - - Additional module-level constants are defined for GNU/glibc based systems. - These are :const:`ST_NODEV` (disallow access to device special files), - :const:`ST_NOEXEC` (disallow program execution), :const:`ST_SYNCHRONOUS` - (writes are synced at once), :const:`ST_MANDLOCK` (allow mandatory locks on an FS), - :const:`ST_WRITE` (write on file/directory/symlink), :const:`ST_APPEND` - (append-only file), :const:`ST_IMMUTABLE` (immutable file), :const:`ST_NOATIME` - (do not update access times), :const:`ST_NODIRATIME` (do not update directory access - times), :const:`ST_RELATIME` (update atime relative to mtime/ctime). + Perform a :manpage:`statvfs(3)` system call on the given path. The return value + is a :class:`statvfs_result` whose attributes describe the filesystem + on the given path and correspond to the members of the :c:struct:`statvfs` + structure. This function can support :ref:`specifying a file descriptor `. .. availability:: Unix. - .. versionchanged:: 3.2 - The :const:`ST_RDONLY` and :const:`ST_NOSUID` constants were added. - .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor. - .. versionchanged:: 3.4 - The :const:`ST_NODEV`, :const:`ST_NOEXEC`, :const:`ST_SYNCHRONOUS`, - :const:`ST_MANDLOCK`, :const:`ST_WRITE`, :const:`ST_APPEND`, - :const:`ST_IMMUTABLE`, :const:`ST_NOATIME`, :const:`ST_NODIRATIME`, - and :const:`ST_RELATIME` constants were added. - .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. versionchanged:: 3.7 - Added the :attr:`f_fsid` attribute. + +.. class:: statvfs_result + + Filesystem statistics returned by :func:`os.statvfs` and :func:`os.fstatvfs`. + See :manpage:`statvfs(3)` for more details. + + .. attribute:: f_bsize + + Block size. + + .. attribute:: f_frsize + + Fragment size. + + .. attribute:: f_blocks + + Number of :attr:`~statvfs_result.f_frsize` sized blocks the filesystem + can contain. + + .. attribute:: f_bfree + + Number of free blocks. + + .. attribute:: f_bavail + + Number of free blocks for unprivileged users. + + .. attribute:: f_files + + Number of file entries, inodes, the filesystem can contain. + + .. attribute:: f_ffree + + Number of free files entries. + + .. attribute:: f_favail + + Number of free file entries for unprivileged users. + + .. attribute:: f_flag + + Bit-mask of mount flags. The following flags are defined: + :data:`ST_RDONLY`, :data:`ST_NOSUID`, :data:`ST_NODEV`, + :data:`ST_NOEXEC`, :data:`ST_SYNCHRONOUS`, :data:`ST_MANDLOCK`, + :data:`ST_WRITE`, :data:`ST_APPEND`, :data:`ST_IMMUTABLE`, + :data:`ST_NOATIME`, :data:`ST_NODIRATIME`, and :data:`ST_RELATIME`. + + .. attribute:: f_namemax + + Filesystem max filename length. OS specific limitations such as + :ref:`Windows MAX_PATH ` and those described in Linux + :manpage:`pathname(7)` may exist. + + .. attribute:: f_fsid + + Filesystem ID. + + .. versionadded:: 3.7 + + +The following flags are used in :attr:`statvfs_result.f_flag`. + +.. data:: ST_RDONLY + + Read-only filesystem. + + .. versionadded:: 3.2 + +.. data:: ST_NOSUID + + Setuid/setgid bits are disabled or not supported. + + .. versionadded:: 3.2 + +.. data:: ST_NODEV + + Disallow access to device special files. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_NOEXEC + + Disallow program execution. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_SYNCHRONOUS + + Writes are synced at once. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_MANDLOCK + + Allow mandatory locks on an FS. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_WRITE + + Write on file/directory/symlink. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_APPEND + + Append-only file. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_IMMUTABLE + + Immutable file. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_NOATIME + + Do not update access times. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_NODIRATIME + + Do not update directory access times. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_RELATIME + + Update atime relative to mtime/ctime. + + .. availability:: Linux. + + .. versionadded:: 3.4 .. data:: supports_dir_fd - A :class:`set` object indicating which functions in the :mod:`os` + A :class:`set` object indicating which functions in the :mod:`!os` module accept an open file descriptor for their *dir_fd* parameter. Different platforms provide different features, and the underlying functionality Python uses to implement the *dir_fd* parameter is not @@ -3458,7 +4023,7 @@ features: .. data:: supports_fd A :class:`set` object indicating which functions in the - :mod:`os` module permit specifying their *path* parameter as an open file + :mod:`!os` module permit specifying their *path* parameter as an open file descriptor on the local platform. Different platforms provide different features, and the underlying functionality Python uses to accept open file descriptors as *path* arguments is not available on all platforms Python @@ -3477,7 +4042,7 @@ features: .. data:: supports_follow_symlinks - A :class:`set` object indicating which functions in the :mod:`os` module + A :class:`set` object indicating which functions in the :mod:`!os` module accept ``False`` for their *follow_symlinks* parameter on the local platform. Different platforms provide different features, and the underlying functionality Python uses to implement *follow_symlinks* is not available @@ -3502,6 +4067,9 @@ features: Create a symbolic link pointing to *src* named *dst*. + The *src* parameter refers to the target of the link (the file or directory being linked to), + and *dst* is the name of the link being created. + On Windows, a symlink represents either a file or a directory, and does not morph to the target dynamically. If the target is present, the type of the symlink will be created to match. Otherwise, the symlink will be created @@ -3600,7 +4168,8 @@ features: where each member is an int expressing nanoseconds. - If *times* is not ``None``, it must be a 2-tuple of the form ``(atime, mtime)`` - where each member is an int or float expressing seconds. + where each member is a real number expressing seconds, + rounded down to nanoseconds. - If *times* is ``None`` and *ns* is unspecified, this is equivalent to specifying ``ns=(atime_ns, mtime_ns)`` where both times are the current time. @@ -3627,6 +4196,9 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.15 + Accepts any real numbers as *times*, not only integers or floats. + .. function:: walk(top, topdown=True, onerror=None, followlinks=False) @@ -3757,9 +4329,9 @@ features: import os for root, dirs, files, rootfd in os.fwalk('python/Lib/xml'): - print(root, "consumes", end="") + print(root, "consumes", end=" ") print(sum([os.stat(name, dir_fd=rootfd).st_size for name in files]), - end="") + end=" ") print("bytes in", len(files), "non-directory files") if '__pycache__' in dirs: dirs.remove('__pycache__') # don't visit __pycache__ directories @@ -3873,7 +4445,7 @@ features: import os # semaphore with start value '1' - fd = os.eventfd(1, os.EFD_SEMAPHORE | os.EFC_CLOEXEC) + fd = os.eventfd(1, os.EFD_SEMAPHORE | os.EFD_CLOEXEC) try: # acquire semaphore v = os.eventfd_read(fd) @@ -3982,7 +4554,7 @@ Naturally, they are all only available on Linux. except it includes any time that the system is suspended. The file descriptor's behaviour can be modified by specifying a *flags* value. - Any of the following variables may used, combined using bitwise OR + Any of the following variables may be used, combined using bitwise OR (the ``|`` operator): - :const:`TFD_NONBLOCK` @@ -4014,7 +4586,7 @@ Naturally, they are all only available on Linux. *fd* must be a valid timer file descriptor. The timer's behaviour can be modified by specifying a *flags* value. - Any of the following variables may used, combined using bitwise OR + Any of the following variables may be used, combined using bitwise OR (the ``|`` operator): - :const:`TFD_TIMER_ABSTIME` @@ -4032,7 +4604,7 @@ Naturally, they are all only available on Linux. the timer will fire when the timer's clock (set by *clockid* in :func:`timerfd_create`) reaches *initial* seconds. - The timer's interval is set by the *interval* :py:class:`float`. + The timer's interval is set by the *interval* real number. If *interval* is zero, the timer only fires once, on the initial expiration. If *interval* is greater than zero, the timer fires every time *interval* seconds have elapsed since the previous expiration. @@ -4083,7 +4655,7 @@ Naturally, they are all only available on Linux. Return a two-item tuple of floats (``next_expiration``, ``interval``). - ``next_expiration`` denotes the relative time until next the timer next fires, + ``next_expiration`` denotes the relative time until the timer next fires, regardless of if the :const:`TFD_TIMER_ABSTIME` flag is set. ``interval`` denotes the timer's interval. @@ -4185,6 +4757,10 @@ These functions are all available on Linux only. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.15 + ``os.listxattr(-1)`` now fails with ``OSError(errno.EBADF)`` rather than + listing extended attributes of the current directory. + .. function:: removexattr(path, attribute, *, follow_symlinks=True) @@ -4263,10 +4839,10 @@ to be ignored. .. function:: abort() - Generate a :const:`SIGABRT` signal to the current process. On Unix, the default + Generate a :const:`~signal.SIGABRT` signal to the current process. On Unix, the default behavior is to produce a core dump; on Windows, the process immediately returns an exit code of ``3``. Be aware that calling this function will not call the - Python signal handler registered for :const:`SIGABRT` with + Python signal handler registered for :const:`~signal.SIGABRT` with :func:`signal.signal`. @@ -4319,7 +4895,7 @@ to be ignored. The current process is replaced immediately. Open file objects and descriptors are not flushed, so if there may be data buffered on these open files, you should flush them using - :func:`sys.stdout.flush` or :func:`os.fsync` before calling an + :func:`~io.IOBase.flush` or :func:`os.fsync` before calling an :func:`exec\* ` function. The "l" and "v" variants of the :func:`exec\* ` functions differ in how @@ -4570,6 +5146,8 @@ written in Python, such as a mail server's external command delivery program. master end of the pseudo-terminal. For a more portable approach, use the :mod:`pty` module. If an error occurs :exc:`OSError` is raised. + The returned file descriptor *fd* is :ref:`non-inheritable `. + .. audit-event:: os.forkpty "" os.forkpty .. warning:: @@ -4586,6 +5164,9 @@ written in Python, such as a mail server's external command delivery program. threads, this now raises a :exc:`DeprecationWarning`. See the longer explanation on :func:`os.fork`. + .. versionchanged:: 3.15 + The returned file descriptor is now made non-inheritable. + .. availability:: Unix, not WASI, not Android, not iOS. @@ -4656,13 +5237,25 @@ written in Python, such as a mail server's external command delivery program. .. availability:: Linux >= 5.10 .. versionadded:: 3.12 +.. function:: pidfd_getfd(pidfd, targetfd, *, flags=0) + + Duplicate *targetfd* from the process referred to by the process file + descriptor *pidfd*, into the calling process. The returned file descriptor + is :ref:`non-inheritable `. + + *flags* is reserved, and currently must be ``0``. + + See the :manpage:`pidfd_getfd(2)` man page for more details. + + .. availability:: Linux >= 5.6, Android >= :func:`build-time ` API level 31 + .. versionadded:: next .. function:: plock(op, /) Lock program segments into memory. The value of *op* (defined in ````) determines which segments are locked. - .. availability:: Unix, not WASI, not iOS. + .. availability:: Unix, not WASI, not macOS, not iOS. .. function:: popen(cmd, mode='r', buffering=-1) @@ -4704,9 +5297,8 @@ written in Python, such as a mail server's external command delivery program. Use :class:`subprocess.Popen` or :func:`subprocess.run` to control options like encodings. - .. deprecated:: 3.14 - The function is :term:`soft deprecated` and should no longer be used to - write new code. The :mod:`subprocess` module is recommended instead. + .. soft-deprecated:: 3.14 + The :mod:`subprocess` module is recommended instead. .. function:: posix_spawn(path, argv, env, *, file_actions=None, \ @@ -4934,9 +5526,8 @@ written in Python, such as a mail server's external command delivery program. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. deprecated:: 3.14 - These functions are :term:`soft deprecated` and should no longer be used - to write new code. The :mod:`subprocess` module is recommended instead. + .. soft-deprecated:: 3.14 + The :mod:`subprocess` module is recommended instead. .. data:: P_NOWAIT @@ -5610,7 +6201,7 @@ Miscellaneous System Information .. versionchanged:: 3.13 If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, - :func:`cpu_count` returns the overridden value *n*. + :func:`cpu_count` returns the override value *n*. .. function:: getloadavg() @@ -5632,7 +6223,7 @@ Miscellaneous System Information in the **system**. If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, - :func:`process_cpu_count` returns the overridden value *n*. + :func:`process_cpu_count` returns the override value *n*. See also the :func:`sched_getaffinity` function. diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index ebf5756146df925..ab92c142c37a4f6 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -311,7 +311,7 @@ Pure paths provide the following methods and properties: .. attribute:: PurePath.parser The implementation of the :mod:`os.path` module used for low-level path - parsing and joining: either :mod:`posixpath` or :mod:`ntpath`. + parsing and joining: either :mod:`!posixpath` or :mod:`!ntpath`. .. versionadded:: 3.13 @@ -486,6 +486,10 @@ Pure paths provide the following methods and properties: >>> PurePosixPath('my/library').stem 'library' + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. + .. method:: PurePath.as_posix() @@ -1262,6 +1266,8 @@ Reading and writing files >>> p.read_text() 'Text file contents' + Return the number of characters written. + An existing file of the same name is overwritten. The optional parameters have the same meaning as in :func:`open`. @@ -1282,6 +1288,8 @@ Reading and writing files >>> p.read_bytes() b'Binary file contents' + Return the number of bytes written. + An existing file of the same name is overwritten. .. versionadded:: 3.5 @@ -1331,6 +1339,10 @@ Reading directories PosixPath('setup.py'), PosixPath('test_pathlib.py')] + .. note:: + The paths are returned in no particular order. + If you need a specific order, sort the results. + .. seealso:: :ref:`pathlib-pattern-language` documentation. @@ -1343,6 +1355,11 @@ Reading directories ``False``, this method follows symlinks except when expanding "``**``" wildcards. Set *recurse_symlinks* to ``True`` to always follow symlinks. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob .. versionchanged:: 3.12 @@ -1365,6 +1382,15 @@ Reading directories Glob the given relative *pattern* recursively. This is like calling :func:`Path.glob` with "``**/``" added in front of the *pattern*. + .. note:: + The paths are returned in no particular order. + If you need a specific order, sort the results. + + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. seealso:: :ref:`pathlib-pattern-language` and :meth:`Path.glob` documentation. @@ -1492,7 +1518,8 @@ Creating files and directories :meth:`~Path.write_bytes` methods are often used to create files. -.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False) +.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False, *, \ + parent_mode=None) Create a new directory at this given path. If *mode* is given, it is combined with the process's ``umask`` value to determine the file mode @@ -1503,6 +1530,12 @@ Creating files and directories as needed; they are created with the default permissions without taking *mode* into account (mimicking the POSIX ``mkdir -p`` command). + If *parent_mode* is not ``None``, it is used as the mode for any + newly-created, intermediate-level directories when *parents* is true. + Like *mode*, it is combined with the process's ``umask`` value. + Otherwise, intermediate directories are created with the default + permissions (also subject to the umask). + If *parents* is false (the default), a missing parent raises :exc:`FileNotFoundError`. @@ -1516,6 +1549,9 @@ Creating files and directories .. versionchanged:: 3.5 The *exist_ok* parameter was added. + .. versionadded:: 3.15 + The *parent_mode* parameter. + .. method:: Path.symlink_to(target, target_is_directory=False) @@ -1875,7 +1911,7 @@ Below is a table mapping various :mod:`os` functions to their corresponding :class:`PurePath`/:class:`Path` equivalent. ===================================== ============================================== -:mod:`os` and :mod:`os.path` :mod:`pathlib` +:mod:`os` and :mod:`os.path` :mod:`!pathlib` ===================================== ============================================== :func:`os.path.dirname` :attr:`PurePath.parent` :func:`os.path.basename` :attr:`PurePath.name` @@ -1934,7 +1970,7 @@ Protocols :synopsis: pathlib types for static type checking -The :mod:`pathlib.types` module provides types for static type checking. +The :mod:`!pathlib.types` module provides types for static type checking. .. versionadded:: 3.14 diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index f4b51664545be58..bfe017a5c8fe1c7 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -1,7 +1,7 @@ .. _debugger: -:mod:`pdb` --- The Python Debugger -================================== +:mod:`!pdb` --- The Python Debugger +=================================== .. module:: pdb :synopsis: The Python debugger for interactive interpreters. @@ -12,7 +12,7 @@ -------------- -The module :mod:`pdb` defines an interactive source code debugger for Python +The module :mod:`!pdb` defines an interactive source code debugger for Python programs. It supports setting (conditional) breakpoints and single stepping at the source line level, inspection of stack frames, source code listing, and evaluation of arbitrary Python code in the context of any stack frame. It also @@ -75,9 +75,14 @@ The debugger's prompt is ``(Pdb)``, which is the indicator that you are in debug arguments of the ``p`` command. +.. _pdb-cli: + +Command-line interface +---------------------- + .. program:: pdb -You can also invoke :mod:`pdb` from the command line to debug other scripts. For +You can also invoke :mod:`!pdb` from the command line to debug other scripts. For example:: python -m pdb [-c command] (-m module | -p pid | pyfile) [args ...] @@ -333,7 +338,7 @@ access further features, you have to do this yourself: .. _debugger-commands: -Debugger Commands +Debugger commands ----------------- The commands recognized by the debugger are listed below. Most commands can be @@ -515,7 +520,8 @@ can be overridden by the local file. To remove all commands from a breakpoint, type ``commands`` and follow it immediately with ``end``; that is, give no commands. - With no *bpnumber* argument, ``commands`` refers to the last breakpoint set. + With no *bpnumber* argument, ``commands`` refers to the most recently set + breakpoint that still exists. You can use breakpoint commands to start your program up again. Simply use the :pdbcmd:`continue` command, or :pdbcmd:`step`, diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 007c9fe1b950cf2..8eadc2cf2b1ef0d 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -4,9 +4,6 @@ .. module:: pickle :synopsis: Convert Python objects to streams of bytes and back. -.. sectionauthor:: Jim Kerr . -.. sectionauthor:: Barry Warsaw - **Source code:** :source:`Lib/pickle.py` .. index:: @@ -19,7 +16,7 @@ -------------- -The :mod:`pickle` module implements binary protocols for serializing and +The :mod:`!pickle` module implements binary protocols for serializing and de-serializing a Python object structure. *"Pickling"* is the process whereby a Python object hierarchy is converted into a byte stream, and *"unpickling"* is the inverse operation, whereby a byte stream @@ -50,35 +47,22 @@ Comparison with ``marshal`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Python has a more primitive serialization module called :mod:`marshal`, but in -general :mod:`pickle` should always be the preferred way to serialize Python +general :mod:`!pickle` should always be the preferred way to serialize Python objects. :mod:`marshal` exists primarily to support Python's :file:`.pyc` files. -The :mod:`pickle` module differs from :mod:`marshal` in several significant ways: - -* The :mod:`pickle` module keeps track of the objects it has already serialized, - so that later references to the same object won't be serialized again. - :mod:`marshal` doesn't do this. - - This has implications both for recursive objects and object sharing. Recursive - objects are objects that contain references to themselves. These are not - handled by marshal, and in fact, attempting to marshal recursive objects will - crash your Python interpreter. Object sharing happens when there are multiple - references to the same object in different places in the object hierarchy being - serialized. :mod:`pickle` stores such objects only once, and ensures that all - other references point to the master copy. Shared objects remain shared, which - can be very important for mutable objects. +The :mod:`!pickle` module differs from :mod:`marshal` in several significant ways: * :mod:`marshal` cannot be used to serialize user-defined classes and their - instances. :mod:`pickle` can save and restore class instances transparently, + instances. :mod:`!pickle` can save and restore class instances transparently, however the class definition must be importable and live in the same module as - when the object was stored. + when the object was pickled. * The :mod:`marshal` serialization format is not guaranteed to be portable across Python versions. Because its primary job in life is to support :file:`.pyc` files, the Python implementers reserve the right to change the serialization format in non-backwards compatible ways should the need arise. - The :mod:`pickle` serialization format is guaranteed to be backwards compatible + The :mod:`!pickle` serialization format is guaranteed to be backwards compatible across Python releases provided a compatible pickle protocol is chosen and pickling and unpickling code deals with Python 2 to Python 3 type differences if your data is crossing that unique breaking change language boundary. @@ -123,17 +107,17 @@ Data stream format .. index:: single: External Data Representation -The data format used by :mod:`pickle` is Python-specific. This has the +The data format used by :mod:`!pickle` is Python-specific. This has the advantage that there are no restrictions imposed by external standards such as JSON (which can't represent pointer sharing); however it means that non-Python programs may not be able to reconstruct pickled Python objects. -By default, the :mod:`pickle` data format uses a relatively compact binary +By default, the :mod:`!pickle` data format uses a relatively compact binary representation. If you need optimal size characteristics, you can efficiently :doc:`compress ` pickled data. The module :mod:`pickletools` contains tools for analyzing data streams -generated by :mod:`pickle`. :mod:`pickletools` source code has extensive +generated by :mod:`!pickle`. :mod:`pickletools` source code has extensive comments about opcodes used by pickle protocols. There are currently 6 different protocols which can be used for pickling. @@ -167,9 +151,9 @@ to read the pickle produced. .. note:: Serialization is a more primitive notion than persistence; although - :mod:`pickle` reads and writes file objects, it does not handle the issue of + :mod:`!pickle` reads and writes file objects, it does not handle the issue of naming persistent objects, nor the (even more complicated) issue of concurrent - access to persistent objects. The :mod:`pickle` module can transform a complex + access to persistent objects. The :mod:`!pickle` module can transform a complex object into a byte stream and it can transform the byte stream into an object with the same internal structure. Perhaps the most obvious thing to do with these byte streams is to write them onto a file, but it is also conceivable to @@ -186,7 +170,7 @@ Similarly, to de-serialize a data stream, you call the :func:`loads` function. However, if you want more control over serialization and de-serialization, you can create a :class:`Pickler` or an :class:`Unpickler` object, respectively. -The :mod:`pickle` module provides the following constants: +The :mod:`!pickle` module provides the following constants: .. data:: HIGHEST_PROTOCOL @@ -217,7 +201,7 @@ The :mod:`pickle` module provides the following constants: The default protocol is 5. -The :mod:`pickle` module provides the following functions to make the pickling +The :mod:`!pickle` module provides the following functions to make the pickling process more convenient: .. function:: dump(obj, file, protocol=None, *, fix_imports=True, buffer_callback=None) @@ -275,7 +259,7 @@ process more convenient: The *buffers* argument was added. -The :mod:`pickle` module defines three exceptions: +The :mod:`!pickle` module defines three exceptions: .. exception:: PickleError @@ -300,7 +284,7 @@ The :mod:`pickle` module defines three exceptions: IndexError. -The :mod:`pickle` module exports three classes, :class:`Pickler`, +The :mod:`!pickle` module exports three classes, :class:`Pickler`, :class:`Unpickler` and :class:`PickleBuffer`: .. class:: Pickler(file, protocol=None, *, fix_imports=True, buffer_callback=None) @@ -532,7 +516,7 @@ The following types can be pickled: * classes accessible from the top level of a module; -* instances of such classes whose the result of calling :meth:`~object.__getstate__` +* instances of such classes for which the result of calling :meth:`~object.__getstate__` is picklable (see section :ref:`pickle-inst` for details). Attempts to pickle unpicklable objects will raise the :exc:`PicklingError` @@ -601,7 +585,7 @@ methods: .. method:: object.__getnewargs_ex__() - In protocols 2 and newer, classes that implements the + In protocols 2 and newer, classes that implement the :meth:`__getnewargs_ex__` method can dictate the values passed to the :meth:`__new__` method upon unpickling. The method must return a pair ``(args, kwargs)`` where *args* is a tuple of positional arguments @@ -709,7 +693,10 @@ or both. If a string is returned, the string should be interpreted as the name of a global variable. It should be the object's local name relative to its module; the pickle module searches the module namespace to determine the - object's module. This behaviour is typically useful for singletons. + object's module: for a given ``obj`` to be pickled, the ``__module__`` + attribute is looked up on ``obj`` directly, which falls back to a lookup + on the type of ``obj`` if no ``__module__`` instance attribute is set. + This behaviour is typically useful for singletons. When a tuple is returned, it must be between two and six items long. Optional items can either be omitted, or ``None`` can be provided as their @@ -732,8 +719,8 @@ or both. These items will be appended to the object either using ``obj.append(item)`` or, in batch, using ``obj.extend(list_of_items)``. This is primarily used for list subclasses, but may be used by other - classes as long as they have - :ref:`append and extend methods ` with + classes as long as they have :meth:`~sequence.append` + and :meth:`~sequence.extend` methods with the appropriate signature. (Whether :meth:`!append` or :meth:`!extend` is used depends on which pickle protocol version is used as well as the number of items to append, so both must be supported.) @@ -773,13 +760,13 @@ Persistence of External Objects single: persistent_id (pickle protocol) single: persistent_load (pickle protocol) -For the benefit of object persistence, the :mod:`pickle` module supports the +For the benefit of object persistence, the :mod:`!pickle` module supports the notion of a reference to an object outside the pickled data stream. Such objects are referenced by a persistent ID, which should be either a string of alphanumeric characters (for protocol 0) [#]_ or just an arbitrary object (for any newer protocol). -The resolution of such persistent IDs is not defined by the :mod:`pickle` +The resolution of such persistent IDs is not defined by the :mod:`!pickle` module; it will delegate this resolution to the user-defined methods on the pickler and unpickler, :meth:`~Pickler.persistent_id` and :meth:`~Unpickler.persistent_load` respectively. @@ -973,10 +960,10 @@ Out-of-band Buffers .. versionadded:: 3.8 -In some contexts, the :mod:`pickle` module is used to transfer massive amounts +In some contexts, the :mod:`!pickle` module is used to transfer massive amounts of data. Therefore, it can be important to minimize the number of memory copies, to preserve performance and resource consumption. However, normal -operation of the :mod:`pickle` module, as it transforms a graph-like structure +operation of the :mod:`!pickle` module, as it transforms a graph-like structure of objects into a sequential stream of bytes, intrinsically involves copying data to and from the pickle stream. @@ -995,8 +982,8 @@ for any large data. A :class:`PickleBuffer` object *signals* that the underlying buffer is eligible for out-of-band data transfer. Those objects remain compatible -with normal usage of the :mod:`pickle` module. However, consumers can also -opt-in to tell :mod:`pickle` that they will handle those buffers by +with normal usage of the :mod:`!pickle` module. However, consumers can also +opt-in to tell :mod:`!pickle` that they will handle those buffers by themselves. Consumer API @@ -1172,7 +1159,7 @@ Performance Recent versions of the pickle protocol (from protocol 2 and upwards) feature efficient binary encodings for several common features and built-in types. -Also, the :mod:`pickle` module has a transparent optimizer written in C. +Also, the :mod:`!pickle` module has a transparent optimizer written in C. .. _pickle-example: @@ -1215,7 +1202,7 @@ The following example reads the resulting pickled data. :: Command-line interface ---------------------- -The :mod:`pickle` module can be invoked as a script from the command line, +The :mod:`!pickle` module can be invoked as a script from the command line, it will display contents of the pickle files. However, when the pickle file that you want to examine comes from an untrusted source, ``-m pickletools`` is a safer option because it does not execute pickle bytecode, see @@ -1243,7 +1230,7 @@ The following option is accepted: Tools for working with and analyzing pickled data. Module :mod:`shelve` - Indexed databases of objects; uses :mod:`pickle`. + Indexed databases of objects; uses :mod:`!pickle`. Module :mod:`copy` Shallow and deep object copying. diff --git a/Doc/library/pickletools.rst b/Doc/library/pickletools.rst index 30fc2962e0bf785..769ca60af1837ea 100644 --- a/Doc/library/pickletools.rst +++ b/Doc/library/pickletools.rst @@ -15,7 +15,7 @@ This module contains various constants relating to the intimate details of the few useful functions for analyzing pickled data. The contents of this module are useful for Python core developers who are working on the :mod:`pickle`; ordinary users of the :mod:`pickle` module probably won't find the -:mod:`pickletools` module relevant. +:mod:`!pickletools` module relevant. .. _pickletools-cli: @@ -79,6 +79,9 @@ Command-line options A pickle file to read, or ``-`` to indicate reading from standard input. +.. versionadded:: 3.15 + Output is in color by default and can be + :ref:`controlled using environment variables `. Programmatic interface diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 47d24b6f7d06bbe..5473a367c49a3ab 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -151,26 +151,50 @@ support. :meth:`get_data ` API. The *package* argument should be the name of a package, in standard module format (``foo.bar``). The *resource* argument should be in the form of a relative - filename, using ``/`` as the path separator. The parent directory name - ``..`` is not allowed, and nor is a rooted name (starting with a ``/``). + filename, using ``/`` as the path separator. The function returns a binary string that is the contents of the specified resource. + This function uses the :term:`loader` method + :func:`~importlib.abc.FileLoader.get_data` + to support modules installed in the filesystem, but also in zip files, + databases, or elsewhere. + For packages located in the filesystem, which have already been imported, this is the rough equivalent of:: d = os.path.dirname(sys.modules[package].__file__) data = open(os.path.join(d, resource), 'rb').read() + Like the :func:`open` function, :func:`!get_data` can follow parent + directories (``../``) and absolute paths (starting with ``/`` or ``C:/``, + for example). + It can open compilation/installation artifacts like ``.py`` and ``.pyc`` + files or files with :func:`reserved filenames `. + To be compatible with non-filesystem loaders, avoid using these features. + + .. warning:: + + This function is intended for trusted input. + It does not verify that *resource* "belongs" to *package*. + + If you use a user-provided *resource* path, consider verifying it. + For example, require an alphanumeric filename with a known extension, or + install and check a list of known resources. + If the package cannot be located or loaded, or it uses a :term:`loader` which does not support :meth:`get_data `, then ``None`` is returned. In particular, the :term:`loader` for :term:`namespace packages ` does not support :meth:`get_data `. + .. seealso:: -.. function:: resolve_name(name) + The :mod:`importlib.resources` module provides structured access to + module resources. + +.. function:: resolve_name(name, *, strict=False) Resolve a name to an object. @@ -184,6 +208,7 @@ support. * ``W(.W)*`` * ``W(.W)*:(W(.W)*)?`` + * ``W(.W)*:(W(.W)*)`` The first form is intended for backward compatibility only. It assumes that some part of the dotted name is a package, and the rest is an object @@ -198,6 +223,11 @@ support. hierarchy within that package. Only one import is needed in this form. If it ends with the colon, then a module object is returned. + The first two forms are accepted when ``strict=False`` (the default). + + The third form requires both the module name and callable, separated by + a colon. Only this form is accepted when ``strict=True``. + The function will return an object (which might be a module), or raise one of the following exceptions: @@ -209,3 +239,7 @@ support. hierarchy within the imported package to get to the desired object. .. versionadded:: 3.9 + + .. versionchanged:: 3.15 + + The optional keyword-only ``strict`` flag was added. diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 37df13f8a1eb8cf..1d30966794fd1bf 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -4,9 +4,6 @@ .. module:: platform :synopsis: Retrieves as much platform identifying data as possible. -.. moduleauthor:: Marc-André Lemburg -.. sectionauthor:: Bjorn Pettersen - **Source code:** :source:`Lib/platform.py` -------------- @@ -56,6 +53,8 @@ Cross platform Returns the machine type, e.g. ``'AMD64'``. An empty string is returned if the value cannot be determined. + The output is platform-dependent and may differ in casing and naming conventions. + .. function:: node() @@ -187,6 +186,14 @@ Cross platform .. versionchanged:: 3.9 :attr:`processor` is resolved late instead of immediately. +.. function:: invalidate_caches() + + Clear out the internal cache of information, such as the :func:`uname`. + This is typically useful when the platform's :func:`node` is changed + by an external process and one needs to retrieve the updated value. + + .. versionadded:: 3.14 + Windows platform ---------------- @@ -347,7 +354,7 @@ Android platform Command-line usage ------------------ -:mod:`platform` can also be invoked directly using the :option:`-m` +:mod:`!platform` can also be invoked directly using the :option:`-m` switch of the interpreter:: python -m platform [--terse] [--nonaliased] [{nonaliased,terse} ...] @@ -370,14 +377,3 @@ The following options are accepted: You can also pass one or more positional arguments (``terse``, ``nonaliased``) to explicitly control the output format. These behave similarly to their corresponding options. - -Miscellaneous -------------- - -.. function:: invalidate_caches() - - Clear out the internal cache of information, such as the :func:`uname`. - This is typically useful when the platform's :func:`node` is changed - by an external process and one needs to retrieve the updated value. - - .. versionadded:: 3.14 diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 415c4b45c4f100e..72140e41675c350 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -4,10 +4,6 @@ .. module:: plistlib :synopsis: Generate and parse Apple plist files. -.. moduleauthor:: Jack Jansen -.. sectionauthor:: Georg Brandl -.. (harvested from docstrings in the original file) - **Source code:** :source:`Lib/plistlib.py` .. index:: @@ -22,7 +18,7 @@ and XML plist files. The property list (``.plist``) file format is a simple serialization supporting basic object types, like dictionaries, lists, numbers and strings. Usually the -top level object is a dictionary. +top level object is a dictionary or a frozen dictionary. To write out and to parse a plist file, use the :func:`dump` and :func:`load` functions. @@ -184,7 +180,7 @@ Examples Generating a plist:: - import datetime + import datetime as dt import plistlib pl = dict( @@ -200,7 +196,7 @@ Generating a plist:: ), someData = b"", someMoreData = b"" * 10, - aDate = datetime.datetime.now() + aDate = dt.datetime.now() ) print(plistlib.dumps(pl).decode()) diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst index 23f20b00e6dc6d8..cd3a58016e9c12e 100644 --- a/Doc/library/poplib.rst +++ b/Doc/library/poplib.rst @@ -4,9 +4,6 @@ .. module:: poplib :synopsis: POP3 protocol client (requires sockets). -.. sectionauthor:: Andrew T. Csillag -.. revised by ESR, January 2000 - **Source code:** :source:`Lib/poplib.py` .. index:: pair: POP3; protocol @@ -30,7 +27,7 @@ mailserver supports IMAP, you would be better off using the .. include:: ../includes/wasm-notavail.rst -The :mod:`poplib` module provides two classes: +The :mod:`!poplib` module provides two classes: .. class:: POP3(host, port=POP3_PORT[, timeout]) @@ -86,7 +83,7 @@ The :mod:`poplib` module provides two classes: .. versionchanged:: 3.12 The deprecated *keyfile* and *certfile* parameters have been removed. -One exception is defined as an attribute of the :mod:`poplib` module: +One exception is defined as an attribute of the :mod:`!poplib` module: .. exception:: error_proto diff --git a/Doc/library/posix.rst b/Doc/library/posix.rst index 14ab3e91e8a8e4c..7d43b5d4cf7657e 100644 --- a/Doc/library/posix.rst +++ b/Doc/library/posix.rst @@ -2,7 +2,6 @@ ==================================================== .. module:: posix - :platform: Unix :synopsis: The most common POSIX system calls (normally used via module os). -------------- @@ -17,10 +16,10 @@ interface). **Do not import this module directly.** Instead, import the module :mod:`os`, which provides a *portable* version of this interface. On Unix, the :mod:`os` -module provides a superset of the :mod:`posix` interface. On non-Unix operating -systems the :mod:`posix` module is not available, but a subset is always +module provides a superset of the :mod:`!posix` interface. On non-Unix operating +systems the :mod:`!posix` module is not available, but a subset is always available through the :mod:`os` interface. Once :mod:`os` is imported, there is -*no* performance penalty in using it instead of :mod:`posix`. In addition, +*no* performance penalty in using it instead of :mod:`!posix`. In addition, :mod:`os` provides some additional functionality, such as automatically calling :func:`~os.putenv` when an entry in ``os.environ`` is changed. @@ -37,8 +36,6 @@ Large File Support single: large files single: file; large files -.. sectionauthor:: Steve Clift - Several operating systems (including AIX and Solaris) provide support for files that are larger than 2 GiB from a C programming model where :c:expr:`int` and :c:expr:`long` are 32-bit values. This is typically accomplished @@ -67,7 +64,7 @@ Notable Module Contents ----------------------- In addition to many functions described in the :mod:`os` module documentation, -:mod:`posix` defines the following data item: +:mod:`!posix` defines the following data item: .. data:: environ @@ -91,4 +88,4 @@ In addition to many functions described in the :mod:`os` module documentation, which updates the environment on modification. Note also that updating :data:`os.environ` will render this dictionary obsolete. Use of the :mod:`os` module version of this is recommended over direct access to the - :mod:`posix` module. + :mod:`!posix` module. diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index 2985f31bacb47a5..4f043fbb3a46dff 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -4,14 +4,11 @@ .. module:: pprint :synopsis: Data pretty printer. -.. moduleauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/pprint.py` -------------- -The :mod:`pprint` module provides a capability to "pretty-print" arbitrary +The :mod:`!pprint` module provides a capability to "pretty-print" arbitrary Python data structures in a form which can be used as input to the interpreter. If the formatted structures include objects which are not fundamental Python types, the representation may not be loadable. This may be the case if objects @@ -22,8 +19,6 @@ The formatted representation keeps objects on a single line if it can, and breaks them onto multiple lines if they don't fit within the allowed width, adjustable by the *width* parameter defaulting to 80 characters. -Dictionaries are sorted by key before the display is computed. - .. versionchanged:: 3.9 Added support for pretty-printing :class:`types.SimpleNamespace`. @@ -36,7 +31,8 @@ Functions --------- .. function:: pp(object, stream=None, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=False, underscore_numbers=False) + compact=False, expand=False, sort_dicts=False, \ + underscore_numbers=False) Prints the formatted representation of *object*, followed by a newline. This function may be used in the interactive interpreter @@ -74,6 +70,13 @@ Functions each item of a sequence will be formatted on a separate line, otherwise as many items as will fit within the *width* will be formatted on each output line. + Incompatible with *expand*. + + :param bool expand: + If ``True``, + opening parentheses and brackets will be followed by a newline and the + following content will be indented by one level, similar to + pretty-printed JSON. Incompatible with *compact*. :param bool sort_dicts: If ``True``, dictionaries will be formatted with @@ -100,7 +103,8 @@ Functions .. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) + compact=False, expand=False, sort_dicts=True, \ + underscore_numbers=False) Alias for :func:`~pprint.pp` with *sort_dicts* set to ``True`` by default, which would automatically sort the dictionaries' keys, @@ -108,10 +112,11 @@ Functions .. function:: pformat(object, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) + compact=False, expand=False, sort_dicts=True, \ + underscore_numbers=False) Return the formatted representation of *object* as a string. *indent*, - *width*, *depth*, *compact*, *sort_dicts* and *underscore_numbers* are + *width*, *depth*, *compact*, *expand*, *sort_dicts* and *underscore_numbers* are passed to the :class:`PrettyPrinter` constructor as formatting parameters and their meanings are as described in the documentation above. @@ -155,7 +160,8 @@ PrettyPrinter Objects .. index:: single: ...; placeholder .. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) + compact=False, expand=False, sort_dicts=True, \ + underscore_numbers=False) Construct a :class:`PrettyPrinter` instance. @@ -179,6 +185,22 @@ PrettyPrinter Objects 'knights', 'ni'], 'spam', 'eggs', 'lumberjack', 'knights', 'ni'] + >>> pp = pprint.PrettyPrinter(width=41, expand=True, indent=3) + >>> pp.pprint(stuff) + [ + [ + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni', + ], + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni', + ] >>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', ... ('parrot', ('fresh fruit',)))))))) >>> pp = pprint.PrettyPrinter(depth=6) @@ -198,6 +220,9 @@ PrettyPrinter Objects .. versionchanged:: 3.11 No longer attempts to write to :data:`!sys.stdout` if it is ``None``. + .. versionchanged:: 3.15 + Added the *expand* parameter. + :class:`PrettyPrinter` instances have the following methods: @@ -420,3 +445,72 @@ cannot be split, the specified width will be exceeded:: 'requires_python': None, 'summary': 'A sample Python project', 'version': '1.2.0'} + +Lastly, we can format like pretty-printed JSON with the *expand* parameter. +Best results are achieved with a higher *indent* value:: + + >>> pprint.pp(project_info, indent=4, expand=True) + { + 'author': 'The Python Packaging Authority', + 'author_email': 'pypa-dev@googlegroups.com', + 'bugtrack_url': None, + 'classifiers': [ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Topic :: Software Development :: Build Tools', + ], + 'description': 'A sample Python project\n' + '=======================\n' + '\n' + 'This is the description file for the project.\n' + '\n' + 'The file should use UTF-8 encoding and be written using ReStructured ' + 'Text. It\n' + 'will be used to generate the project webpage on PyPI, and should be ' + 'written for\n' + 'that purpose.\n' + '\n' + 'Typical contents for this file would include an overview of the project, ' + 'basic\n' + 'usage examples, etc. Generally, including the project changelog in here ' + 'is not\n' + 'a good idea, although a simple "What\'s New" section for the most recent ' + 'version\n' + 'may be appropriate.', + 'description_content_type': None, + 'docs_url': None, + 'download_url': 'UNKNOWN', + 'downloads': {'last_day': -1, 'last_month': -1, 'last_week': -1}, + 'dynamic': None, + 'home_page': 'https://github.com/pypa/sampleproject', + 'keywords': 'sample setuptools development', + 'license': 'MIT', + 'license_expression': None, + 'license_files': None, + 'maintainer': None, + 'maintainer_email': None, + 'name': 'sampleproject', + 'package_url': 'https://pypi.org/project/sampleproject/', + 'platform': 'UNKNOWN', + 'project_url': 'https://pypi.org/project/sampleproject/', + 'project_urls': { + 'Download': 'UNKNOWN', + 'Homepage': 'https://github.com/pypa/sampleproject', + }, + 'provides_extra': None, + 'release_url': 'https://pypi.org/project/sampleproject/1.2.0/', + 'requires_dist': None, + 'requires_python': None, + 'summary': 'A sample Python project', + 'version': '1.2.0', + 'yanked': False, + 'yanked_reason': None, + } diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index c5b88c22325404a..f7e85d1598727f9 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -1,523 +1,64 @@ .. _profile: -******************** -The Python Profilers -******************** +**************************************** +:mod:`!profile` --- Pure Python profiler +**************************************** -**Source code:** :source:`Lib/profile.py`, :source:`Lib/pstats.py`, and :source:`Lib/profile/sample.py` +.. module:: profile + :synopsis: Pure Python profiler (deprecated). + :deprecated: --------------- +**Source code:** :source:`Lib/profile.py` -.. _profiler-introduction: +-------------- -Introduction to the profilers -============================= +.. deprecated-removed:: 3.15 3.17 -.. index:: - single: statistical profiling - single: profiling, statistical - single: deterministic profiling - single: profiling, deterministic +The :mod:`!profile` module is deprecated and will be removed in Python 3.17. +Use :mod:`profiling.tracing` instead. -Python provides both :dfn:`statistical profiling` and :dfn:`deterministic profiling` of -Python programs. A :dfn:`profile` is a set of statistics that describes how -often and for how long various parts of the program executed. These statistics -can be formatted into reports via the :mod:`pstats` module. +The :mod:`!profile` module provides a pure Python implementation of a +deterministic profiler. While useful for understanding profiler internals or +extending profiler behavior through subclassing, its pure Python implementation +introduces significant overhead compared to the C-based :mod:`profiling.tracing` +module. -The Python standard library provides three different profiling implementations: +For most profiling tasks, use: -**Statistical Profiler:** +- :mod:`profiling.sampling` for production debugging with zero overhead +- :mod:`profiling.tracing` for development and testing -1. :mod:`profile.sample` provides statistical profiling of running Python processes - using periodic stack sampling. It can attach to any running Python process without - requiring code modification or restart, making it ideal for production debugging. -**Deterministic Profilers:** +Migration +========= -2. :mod:`cProfile` is recommended for development and testing; it's a C extension with - reasonable overhead that makes it suitable for profiling long-running - programs. Based on :mod:`lsprof`, contributed by Brett Rosen and Ted - Czotter. +Migrating from :mod:`!profile` to :mod:`profiling.tracing` is straightforward. +The APIs are compatible:: -3. :mod:`profile`, a pure Python module whose interface is imitated by - :mod:`cProfile`, but which adds significant overhead to profiled programs. - If you're trying to extend the profiler in some way, the task might be easier - with this module. Originally designed and written by Jim Roskind. + # Old (deprecated) + import profile + profile.run('my_function()') -.. note:: + # New (recommended) + import profiling.tracing + profiling.tracing.run('my_function()') - The profiler modules are designed to provide an execution profile for a given - program, not for benchmarking purposes (for that, there is :mod:`timeit` for - reasonably accurate results). This particularly applies to benchmarking - Python code against C code: the profilers introduce overhead for Python code, - but not for C-level functions, and so the C code would seem faster than any - Python one. - -**Profiler Comparison:** - -+-------------------+----------------------+----------------------+----------------------+ -| Feature | Statistical | Deterministic | Deterministic | -| | (``profile.sample``) | (``cProfile``) | (``profile``) | -+===================+======================+======================+======================+ -| **Target** | Running process | Code you run | Code you run | -+-------------------+----------------------+----------------------+----------------------+ -| **Overhead** | Virtually none | Moderate | High | -+-------------------+----------------------+----------------------+----------------------+ -| **Accuracy** | Statistical approx. | Exact call counts | Exact call counts | -+-------------------+----------------------+----------------------+----------------------+ -| **Setup** | Attach to any PID | Instrument code | Instrument code | -+-------------------+----------------------+----------------------+----------------------+ -| **Use Case** | Production debugging | Development/testing | Profiler extension | -+-------------------+----------------------+----------------------+----------------------+ -| **Implementation**| C extension | C extension | Pure Python | -+-------------------+----------------------+----------------------+----------------------+ +For most code, replacing ``import profile`` with ``import profiling.tracing`` +(and using ``profiling.tracing`` instead of ``profile`` throughout) provides +a straightforward migration path. .. note:: - The statistical profiler (:mod:`profile.sample`) is recommended for most production - use cases due to its extremely low overhead and ability to profile running processes - without modification. It can attach to any Python process and collect performance - data with minimal impact on execution speed, making it ideal for debugging - performance issues in live applications. - - -.. _statistical-profiling: - -What Is Statistical Profiling? -============================== - -:dfn:`Statistical profiling` works by periodically interrupting a running -program to capture its current call stack. Rather than monitoring every -function entry and exit like deterministic profilers, it takes snapshots at -regular intervals to build a statistical picture of where the program spends -its time. - -The sampling profiler uses process memory reading (via system calls like -``process_vm_readv`` on Linux, ``vm_read`` on macOS, and ``ReadProcessMemory`` on -Windows) to attach to a running Python process and extract stack trace -information without requiring any code modification or restart of the target -process. This approach provides several key advantages over traditional -profiling methods. - -The fundamental principle is that if a function appears frequently in the -collected stack samples, it is likely consuming significant CPU time. By -analyzing thousands of samples, the profiler can accurately estimate the -relative time spent in different parts of the program. The statistical nature -means that while individual measurements may vary, the aggregate results -converge to represent the true performance characteristics of the application. - -Since statistical profiling operates externally to the target process, it -introduces virtually no overhead to the running program. The profiler process -runs separately and reads the target process memory without interrupting its -execution. This makes it suitable for profiling production systems where -performance impact must be minimized. - -The accuracy of statistical profiling improves with the number of samples -collected. Short-lived functions may be missed or underrepresented, while -long-running functions will be captured proportionally to their execution time. -This characteristic makes statistical profiling particularly effective for -identifying the most significant performance bottlenecks rather than providing -exhaustive coverage of all function calls. - -Statistical profiling excels at answering questions like "which functions -consume the most CPU time?" and "where should I focus optimization efforts?" -rather than "exactly how many times was this function called?" The trade-off -between precision and practicality makes it an invaluable tool for performance -analysis in real-world applications. - -.. _profile-instant: - -Instant User's Manual -===================== - -This section is provided for users that "don't want to read the manual." It -provides a very brief overview, and allows a user to rapidly perform profiling -on an existing application. - -**Statistical Profiling (Recommended for Production):** - -To profile an existing running process:: - - python -m profile.sample 1234 - -To profile with custom settings:: - - python -m profile.sample -i 50 -d 30 1234 - -**Deterministic Profiling (Development/Testing):** - -To profile a function that takes a single argument, you can do:: - - import cProfile - import re - cProfile.run('re.compile("foo|bar")') - -(Use :mod:`profile` instead of :mod:`cProfile` if the latter is not available on -your system.) - -The above action would run :func:`re.compile` and print profile results like -the following:: - - 214 function calls (207 primitive calls) in 0.002 seconds - - Ordered by: cumulative time - - ncalls tottime percall cumtime percall filename:lineno(function) - 1 0.000 0.000 0.002 0.002 {built-in method builtins.exec} - 1 0.000 0.000 0.001 0.001 :1() - 1 0.000 0.000 0.001 0.001 __init__.py:250(compile) - 1 0.000 0.000 0.001 0.001 __init__.py:289(_compile) - 1 0.000 0.000 0.000 0.000 _compiler.py:759(compile) - 1 0.000 0.000 0.000 0.000 _parser.py:937(parse) - 1 0.000 0.000 0.000 0.000 _compiler.py:598(_code) - 1 0.000 0.000 0.000 0.000 _parser.py:435(_parse_sub) - -The first line indicates that 214 calls were monitored. Of those calls, 207 -were :dfn:`primitive`, meaning that the call was not induced via recursion. The -next line: ``Ordered by: cumulative time`` indicates the output is sorted -by the ``cumtime`` values. The column headings include: - -ncalls - for the number of calls. - -tottime - for the total time spent in the given function (and excluding time made in - calls to sub-functions) - -percall - is the quotient of ``tottime`` divided by ``ncalls`` - -cumtime - is the cumulative time spent in this and all subfunctions (from invocation - till exit). This figure is accurate *even* for recursive functions. - -percall - is the quotient of ``cumtime`` divided by primitive calls - -filename:lineno(function) - provides the respective data of each function - -When there are two numbers in the first column (for example ``3/1``), it means -that the function recursed. The second value is the number of primitive calls -and the former is the total number of calls. Note that when the function does -not recurse, these two values are the same, and only the single figure is -printed. - -Instead of printing the output at the end of the profile run, you can save the -results to a file by specifying a filename to the :func:`run` function:: - - import cProfile - import re - cProfile.run('re.compile("foo|bar")', 'restats') - -The :class:`pstats.Stats` class reads profile results from a file and formats -them in various ways. - -.. _sampling-profiler-cli: - -Statistical Profiler Command Line Interface -=========================================== - -.. program:: profile.sample - -The :mod:`profile.sample` module can be invoked as a script to profile running processes:: - - python -m profile.sample [options] PID - -**Basic Usage Examples:** - -Profile process 1234 for 10 seconds with default settings:: - - python -m profile.sample 1234 - -Profile with custom interval and duration, save to file:: - - python -m profile.sample -i 50 -d 30 -o profile.stats 1234 - -Generate collapsed stacks to use with tools like `flamegraph.pl -`_:: - - python -m profile.sample --collapsed 1234 - -Profile all threads, sort by total time:: - - python -m profile.sample -a --sort-tottime 1234 - -Profile with real-time sampling statistics:: - - python -m profile.sample --realtime-stats 1234 - -**Command Line Options:** - -.. option:: PID - - Process ID of the Python process to profile (required) - -.. option:: -i, --interval INTERVAL - - Sampling interval in microseconds (default: 100) - -.. option:: -d, --duration DURATION - - Sampling duration in seconds (default: 10) - -.. option:: -a, --all-threads - - Sample all threads in the process instead of just the main thread - -.. option:: --realtime-stats - - Print real-time sampling statistics during profiling - -.. option:: --pstats - - Generate pstats output (default) - -.. option:: --collapsed - - Generate collapsed stack traces for flamegraphs - -.. option:: -o, --outfile OUTFILE - - Save output to a file - -**Sorting Options (pstats format only):** - -.. option:: --sort-nsamples - - Sort by number of direct samples - -.. option:: --sort-tottime - - Sort by total time - -.. option:: --sort-cumtime - - Sort by cumulative time (default) - -.. option:: --sort-sample-pct - - Sort by sample percentage - -.. option:: --sort-cumul-pct - - Sort by cumulative sample percentage - -.. option:: --sort-nsamples-cumul - - Sort by cumulative samples - -.. option:: --sort-name - - Sort by function name - -.. option:: -l, --limit LIMIT + The ``cProfile`` module remains available as a backward-compatible alias + to :mod:`profiling.tracing`. Existing code using ``import cProfile`` will + continue to work without modification. - Limit the number of rows in the output (default: 15) -.. option:: --no-summary +:mod:`!profile` and :mod:`!profiling.tracing` module reference +============================================================== - Disable the summary section in the output - -**Understanding Statistical Profile Output:** - -The statistical profiler produces output similar to deterministic profilers but with different column meanings:: - - Profile Stats: - nsamples sample% tottime (ms) cumul% cumtime (ms) filename:lineno(function) - 45/67 12.5 23.450 18.6 56.780 mymodule.py:42(process_data) - 23/23 6.4 15.230 6.4 15.230 :0(len) - -**Column Meanings:** - -- **nsamples**: ``direct/cumulative`` - Times function was directly executing / on call stack -- **sample%**: Percentage of total samples where function was directly executing -- **tottime**: Estimated time spent directly in this function -- **cumul%**: Percentage of samples where function was anywhere on call stack -- **cumtime**: Estimated cumulative time including called functions -- **filename:lineno(function)**: Location and name of the function - -.. _profile-cli: - -:mod:`profile.sample` Module Reference -======================================================= - -.. module:: profile.sample - :synopsis: Python statistical profiler. - -This section documents the programmatic interface for the :mod:`profile.sample` module. -For command-line usage, see :ref:`sampling-profiler-cli`. For conceptual information -about statistical profiling, see :ref:`statistical-profiling` - -.. function:: sample(pid, *, sort=2, sample_interval_usec=100, duration_sec=10, filename=None, all_threads=False, limit=None, show_summary=True, output_format="pstats", realtime_stats=False) - - Sample a Python process and generate profiling data. - - This is the main entry point for statistical profiling. It creates a - :class:`SampleProfiler`, collects stack traces from the target process, and - outputs the results in the specified format. - - :param int pid: Process ID of the target Python process - :param int sort: Sort order for pstats output (default: 2 for cumulative time) - :param int sample_interval_usec: Sampling interval in microseconds (default: 100) - :param int duration_sec: Duration to sample in seconds (default: 10) - :param str filename: Output filename (None for stdout/default naming) - :param bool all_threads: Whether to sample all threads (default: False) - :param int limit: Maximum number of functions to display (default: None) - :param bool show_summary: Whether to show summary statistics (default: True) - :param str output_format: Output format - 'pstats' or 'collapsed' (default: 'pstats') - :param bool realtime_stats: Whether to display real-time statistics (default: False) - - :raises ValueError: If output_format is not 'pstats' or 'collapsed' - - Examples:: - - # Basic usage - profile process 1234 for 10 seconds - import profile.sample - profile.sample.sample(1234) - - # Profile with custom settings - profile.sample.sample(1234, duration_sec=30, sample_interval_usec=50, all_threads=True) - - # Generate collapsed stack traces for flamegraph.pl - profile.sample.sample(1234, output_format='collapsed', filename='profile.collapsed') - -.. class:: SampleProfiler(pid, sample_interval_usec, all_threads) - - Low-level API for the statistical profiler. - - This profiler uses periodic stack sampling to collect performance data - from running Python processes with minimal overhead. It can attach to - any Python process by PID and collect stack traces at regular intervals. - - :param int pid: Process ID of the target Python process - :param int sample_interval_usec: Sampling interval in microseconds - :param bool all_threads: Whether to sample all threads or just the main thread - - .. method:: sample(collector, duration_sec=10) - - Sample the target process for the specified duration. - - Collects stack traces from the target process at regular intervals - and passes them to the provided collector for processing. - - :param collector: Object that implements ``collect()`` method to process stack traces - :param int duration_sec: Duration to sample in seconds (default: 10) - - The method tracks sampling statistics and can display real-time - information if realtime_stats is enabled. - -.. seealso:: - - :ref:`sampling-profiler-cli` - Command-line interface documentation for the statistical profiler. - -Deterministic Profiler Command Line Interface -============================================= - -.. program:: cProfile - -The files :mod:`cProfile` and :mod:`profile` can also be invoked as a script to -profile another script. For example:: - - python -m cProfile [-o output_file] [-s sort_order] (-m module | myscript.py) - -.. option:: -o - - Writes the profile results to a file instead of to stdout. - -.. option:: -s - - Specifies one of the :func:`~pstats.Stats.sort_stats` sort values - to sort the output by. - This only applies when :option:`-o ` is not supplied. - -.. option:: -m - - Specifies that a module is being profiled instead of a script. - - .. versionadded:: 3.7 - Added the ``-m`` option to :mod:`cProfile`. - - .. versionadded:: 3.8 - Added the ``-m`` option to :mod:`profile`. - -The :mod:`pstats` module's :class:`~pstats.Stats` class has a variety of methods -for manipulating and printing the data saved into a profile results file:: - - import pstats - from pstats import SortKey - p = pstats.Stats('restats') - p.strip_dirs().sort_stats(-1).print_stats() - -The :meth:`~pstats.Stats.strip_dirs` method removed the extraneous path from all -the module names. The :meth:`~pstats.Stats.sort_stats` method sorted all the -entries according to the standard module/line/name string that is printed. The -:meth:`~pstats.Stats.print_stats` method printed out all the statistics. You -might try the following sort calls:: - - p.sort_stats(SortKey.NAME) - p.print_stats() - -The first call will actually sort the list by function name, and the second call -will print out the statistics. The following are some interesting calls to -experiment with:: - - p.sort_stats(SortKey.CUMULATIVE).print_stats(10) - -This sorts the profile by cumulative time in a function, and then only prints -the ten most significant lines. If you want to understand what algorithms are -taking time, the above line is what you would use. - -If you were looking to see what functions were looping a lot, and taking a lot -of time, you would do:: - - p.sort_stats(SortKey.TIME).print_stats(10) - -to sort according to time spent within each function, and then print the -statistics for the top ten functions. - -You might also try:: - - p.sort_stats(SortKey.FILENAME).print_stats('__init__') - -This will sort all the statistics by file name, and then print out statistics -for only the class init methods (since they are spelled with ``__init__`` in -them). As one final example, you could try:: - - p.sort_stats(SortKey.TIME, SortKey.CUMULATIVE).print_stats(.5, 'init') - -This line sorts statistics with a primary key of time, and a secondary key of -cumulative time, and then prints out some of the statistics. To be specific, the -list is first culled down to 50% (re: ``.5``) of its original size, then only -lines containing ``init`` are maintained, and that sub-sub-list is printed. - -If you wondered what functions called the above functions, you could now (``p`` -is still sorted according to the last criteria) do:: - - p.print_callers(.5, 'init') - -and you would get a list of callers for each of the listed functions. - -If you want more functionality, you're going to have to read the manual, or -guess what the following functions do:: - - p.print_callees() - p.add('restats') - -Invoked as a script, the :mod:`pstats` module is a statistics browser for -reading and examining profile dumps. It has a simple line-oriented interface -(implemented using :mod:`cmd`) and interactive help. - -:mod:`profile` and :mod:`cProfile` Module Reference -======================================================= - -.. module:: cProfile -.. module:: profile - :synopsis: Python source profiler. - -Both the :mod:`profile` and :mod:`cProfile` modules provide the following -functions: +Both the :mod:`!profile` and :mod:`profiling.tracing` modules provide the +following functions: .. function:: run(command, filename=None, sort=-1) @@ -545,7 +86,7 @@ functions: .. class:: Profile(timer=None, timeunit=0.0, subcalls=True, builtins=True) This class is normally only used if more precise control over profiling is - needed than what the :func:`cProfile.run` function provides. + needed than what the :func:`profiling.tracing.run` function provides. A custom timer can be supplied for measuring how long code takes to run via the *timer* argument. This must be a function that returns a single number @@ -557,9 +98,12 @@ functions: Directly using the :class:`Profile` class allows formatting profile results without writing the profile data to a file:: - import cProfile, pstats, io + import profiling.tracing + import pstats + import io from pstats import SortKey - pr = cProfile.Profile() + + pr = profiling.tracing.Profile() pr.enable() # ... do something ... pr.disable() @@ -570,11 +114,12 @@ functions: print(s.getvalue()) The :class:`Profile` class can also be used as a context manager (supported - only in :mod:`cProfile` module. see :ref:`typecontextmanager`):: + only in :mod:`profiling.tracing`, not in the deprecated :mod:`!profile` + module; see :ref:`typecontextmanager`):: - import cProfile + import profiling.tracing - with cProfile.Profile() as pr: + with profiling.tracing.Profile() as pr: # ... do something ... pr.print_stats() @@ -584,11 +129,11 @@ functions: .. method:: enable() - Start collecting profiling data. Only in :mod:`cProfile`. + Start collecting profiling data. Only in :mod:`profiling.tracing`. .. method:: disable() - Stop collecting profiling data. Only in :mod:`cProfile`. + Stop collecting profiling data. Only in :mod:`profiling.tracing`. .. method:: create_stats() @@ -602,7 +147,7 @@ functions: The *sort* parameter specifies the sorting order of the displayed statistics. It accepts a single key or a tuple of keys to enable - multi-level sorting, as in :func:`Stats.sort_stats `. + multi-level sorting, as in :meth:`pstats.Stats.sort_stats`. .. versionadded:: 3.13 :meth:`~Profile.print_stats` now accepts a tuple of keys. @@ -629,237 +174,43 @@ returns. If the interpreter is terminated (e.g. via a :func:`sys.exit` call during the called command/function execution) no profiling results will be printed. -.. _profile-stats: -The :class:`Stats` Class -======================== +Differences from :mod:`!profiling.tracing` +========================================== -Analysis of the profiler data is done using the :class:`~pstats.Stats` class. +The :mod:`!profile` module differs from :mod:`profiling.tracing` in several +ways: -.. module:: pstats - :synopsis: Statistics object for use with the profiler. +**Higher overhead.** The pure Python implementation is significantly slower +than the C implementation, making it unsuitable for profiling long-running +programs or performance-sensitive code. -.. class:: Stats(*filenames or profile, stream=sys.stdout) +**Calibration support.** The :mod:`!profile` module supports calibration to +compensate for profiling overhead. This is not needed in :mod:`profiling.tracing` +because the C implementation has negligible overhead. - This class constructor creates an instance of a "statistics object" from a - *filename* (or list of filenames) or from a :class:`Profile` instance. Output - will be printed to the stream specified by *stream*. +**Custom timers.** Both modules support custom timers, but :mod:`!profile` +accepts timer functions that return tuples (like :func:`os.times`), while +:mod:`profiling.tracing` requires a function returning a single number. - The file selected by the above constructor must have been created by the - corresponding version of :mod:`profile` or :mod:`cProfile`. To be specific, - there is *no* file compatibility guaranteed with future versions of this - profiler, and there is no compatibility with files produced by other - profilers, or the same profiler run on a different operating system. If - several files are provided, all the statistics for identical functions will - be coalesced, so that an overall view of several processes can be considered - in a single report. If additional files need to be combined with data in an - existing :class:`~pstats.Stats` object, the :meth:`~pstats.Stats.add` method - can be used. +**Subclassing.** The pure Python implementation is easier to subclass and +extend for custom profiling behavior. - Instead of reading the profile data from a file, a :class:`cProfile.Profile` - or :class:`profile.Profile` object can be used as the profile data source. - - :class:`Stats` objects have the following methods: - - .. method:: strip_dirs() - - This method for the :class:`Stats` class removes all leading path - information from file names. It is very useful in reducing the size of - the printout to fit within (close to) 80 columns. This method modifies - the object, and the stripped information is lost. After performing a - strip operation, the object is considered to have its entries in a - "random" order, as it was just after object initialization and loading. - If :meth:`~pstats.Stats.strip_dirs` causes two function names to be - indistinguishable (they are on the same line of the same filename, and - have the same function name), then the statistics for these two entries - are accumulated into a single entry. - - - .. method:: add(*filenames) - - This method of the :class:`Stats` class accumulates additional profiling - information into the current profiling object. Its arguments should refer - to filenames created by the corresponding version of :func:`profile.run` - or :func:`cProfile.run`. Statistics for identically named (re: file, line, - name) functions are automatically accumulated into single function - statistics. - - - .. method:: dump_stats(filename) - - Save the data loaded into the :class:`Stats` object to a file named - *filename*. The file is created if it does not exist, and is overwritten - if it already exists. This is equivalent to the method of the same name - on the :class:`profile.Profile` and :class:`cProfile.Profile` classes. - - - .. method:: sort_stats(*keys) - - This method modifies the :class:`Stats` object by sorting it according to - the supplied criteria. The argument can be either a string or a SortKey - enum identifying the basis of a sort (example: ``'time'``, ``'name'``, - ``SortKey.TIME`` or ``SortKey.NAME``). The SortKey enums argument have - advantage over the string argument in that it is more robust and less - error prone. - - When more than one key is provided, then additional keys are used as - secondary criteria when there is equality in all keys selected before - them. For example, ``sort_stats(SortKey.NAME, SortKey.FILE)`` will sort - all the entries according to their function name, and resolve all ties - (identical function names) by sorting by file name. - - For the string argument, abbreviations can be used for any key names, as - long as the abbreviation is unambiguous. - - The following are the valid string and SortKey: - - +------------------+---------------------+----------------------+ - | Valid String Arg | Valid enum Arg | Meaning | - +==================+=====================+======================+ - | ``'calls'`` | SortKey.CALLS | call count | - +------------------+---------------------+----------------------+ - | ``'cumulative'`` | SortKey.CUMULATIVE | cumulative time | - +------------------+---------------------+----------------------+ - | ``'cumtime'`` | N/A | cumulative time | - +------------------+---------------------+----------------------+ - | ``'file'`` | N/A | file name | - +------------------+---------------------+----------------------+ - | ``'filename'`` | SortKey.FILENAME | file name | - +------------------+---------------------+----------------------+ - | ``'module'`` | N/A | file name | - +------------------+---------------------+----------------------+ - | ``'ncalls'`` | N/A | call count | - +------------------+---------------------+----------------------+ - | ``'pcalls'`` | SortKey.PCALLS | primitive call count | - +------------------+---------------------+----------------------+ - | ``'line'`` | SortKey.LINE | line number | - +------------------+---------------------+----------------------+ - | ``'name'`` | SortKey.NAME | function name | - +------------------+---------------------+----------------------+ - | ``'nfl'`` | SortKey.NFL | name/file/line | - +------------------+---------------------+----------------------+ - | ``'stdname'`` | SortKey.STDNAME | standard name | - +------------------+---------------------+----------------------+ - | ``'time'`` | SortKey.TIME | internal time | - +------------------+---------------------+----------------------+ - | ``'tottime'`` | N/A | internal time | - +------------------+---------------------+----------------------+ - - Note that all sorts on statistics are in descending order (placing most - time consuming items first), where as name, file, and line number searches - are in ascending order (alphabetical). The subtle distinction between - ``SortKey.NFL`` and ``SortKey.STDNAME`` is that the standard name is a - sort of the name as printed, which means that the embedded line numbers - get compared in an odd way. For example, lines 3, 20, and 40 would (if - the file names were the same) appear in the string order 20, 3 and 40. - In contrast, ``SortKey.NFL`` does a numeric compare of the line numbers. - In fact, ``sort_stats(SortKey.NFL)`` is the same as - ``sort_stats(SortKey.NAME, SortKey.FILENAME, SortKey.LINE)``. - - For backward-compatibility reasons, the numeric arguments ``-1``, ``0``, - ``1``, and ``2`` are permitted. They are interpreted as ``'stdname'``, - ``'calls'``, ``'time'``, and ``'cumulative'`` respectively. If this old - style format (numeric) is used, only one sort key (the numeric key) will - be used, and additional arguments will be silently ignored. - - .. For compatibility with the old profiler. - - .. versionadded:: 3.7 - Added the SortKey enum. - - .. method:: reverse_order() - - This method for the :class:`Stats` class reverses the ordering of the - basic list within the object. Note that by default ascending vs - descending order is properly selected based on the sort key of choice. - - .. This method is provided primarily for compatibility with the old - profiler. - - - .. method:: print_stats(*restrictions) - - This method for the :class:`Stats` class prints out a report as described - in the :func:`profile.run` definition. - - The order of the printing is based on the last - :meth:`~pstats.Stats.sort_stats` operation done on the object (subject to - caveats in :meth:`~pstats.Stats.add` and - :meth:`~pstats.Stats.strip_dirs`). - - The arguments provided (if any) can be used to limit the list down to the - significant entries. Initially, the list is taken to be the complete set - of profiled functions. Each restriction is either an integer (to select a - count of lines), or a decimal fraction between 0.0 and 1.0 inclusive (to - select a percentage of lines), or a string that will interpreted as a - regular expression (to pattern match the standard name that is printed). - If several restrictions are provided, then they are applied sequentially. - For example:: - - print_stats(.1, 'foo:') - - would first limit the printing to first 10% of list, and then only print - functions that were part of filename :file:`.\*foo:`. In contrast, the - command:: - - print_stats('foo:', .1) - - would limit the list to all functions having file names :file:`.\*foo:`, - and then proceed to only print the first 10% of them. - - - .. method:: print_callers(*restrictions) - - This method for the :class:`Stats` class prints a list of all functions - that called each function in the profiled database. The ordering is - identical to that provided by :meth:`~pstats.Stats.print_stats`, and the - definition of the restricting argument is also identical. Each caller is - reported on its own line. The format differs slightly depending on the - profiler that produced the stats: - - * With :mod:`profile`, a number is shown in parentheses after each caller - to show how many times this specific call was made. For convenience, a - second non-parenthesized number repeats the cumulative time spent in the - function at the right. - - * With :mod:`cProfile`, each caller is preceded by three numbers: the - number of times this specific call was made, and the total and - cumulative times spent in the current function while it was invoked by - this specific caller. - - - .. method:: print_callees(*restrictions) - - This method for the :class:`Stats` class prints a list of all function - that were called by the indicated function. Aside from this reversal of - direction of calls (re: called vs was called by), the arguments and - ordering are identical to the :meth:`~pstats.Stats.print_callers` method. - - - .. method:: get_stats_profile() - - This method returns an instance of StatsProfile, which contains a mapping - of function names to instances of FunctionProfile. Each FunctionProfile - instance holds information related to the function's profile such as how - long the function took to run, how many times it was called, etc... - - .. versionadded:: 3.9 - Added the following dataclasses: StatsProfile, FunctionProfile. - Added the following function: get_stats_profile. .. _deterministic-profiling: -What Is Deterministic Profiling? +What is deterministic profiling? ================================ :dfn:`Deterministic profiling` is meant to reflect the fact that all *function call*, *function return*, and *exception* events are monitored, and precise timings are made for the intervals between these events (during which time the user's code is executing). In contrast, :dfn:`statistical profiling` (which is -provided by the :mod:`profile.sample` module) periodically samples the effective instruction pointer, and -deduces where time is being spent. The latter technique traditionally involves -less overhead (as the code does not need to be instrumented), but provides only -relative indications of where time is being spent. +provided by the :mod:`profiling.sampling` module) periodically samples the +effective instruction pointer, and deduces where time is being spent. The +latter technique traditionally involves less overhead (as the code does not +need to be instrumented), but provides only relative indications of where time +is being spent. In Python, since there is an interpreter active during execution, the presence of instrumented code is not required in order to do deterministic profiling. @@ -903,15 +254,16 @@ this error. The error that accumulates in this fashion is typically less than the accuracy of the clock (less than one clock tick), but it *can* accumulate and become very significant. -The problem is more important with :mod:`profile` than with the lower-overhead -:mod:`cProfile`. For this reason, :mod:`profile` provides a means of -calibrating itself for a given platform so that this error can be -probabilistically (on the average) removed. After the profiler is calibrated, it -will be more accurate (in a least square sense), but it will sometimes produce -negative numbers (when call counts are exceptionally low, and the gods of -probability work against you :-). ) Do *not* be alarmed by negative numbers in -the profile. They should *only* appear if you have calibrated your profiler, -and the results are actually better than without calibration. +The problem is more important with the deprecated :mod:`!profile` module than +with the lower-overhead :mod:`profiling.tracing`. For this reason, +:mod:`!profile` provides a means of calibrating itself for a given platform so +that this error can be probabilistically (on the average) removed. After the +profiler is calibrated, it will be more accurate (in a least square sense), but +it will sometimes produce negative numbers (when call counts are exceptionally +low, and the gods of probability work against you :-). ) Do *not* be alarmed +by negative numbers in the profile. They should *only* appear if you have +calibrated your profiler, and the results are actually better than without +calibration. .. _profile-calibration: @@ -919,7 +271,7 @@ and the results are actually better than without calibration. Calibration =========== -The profiler of the :mod:`profile` module subtracts a constant from each event +The profiler of the :mod:`!profile` module subtracts a constant from each event handling time to compensate for the overhead of calling the time function, and socking away the results. By default, the constant is 0. The following procedure can be used to obtain a better constant for a given platform (see @@ -957,6 +309,7 @@ When you have a consistent answer, there are three ways you can use it:: If you have a choice, you are better off choosing a smaller constant, and then your results will "less often" show up as negative in profile statistics. + .. _profile-timers: Using a custom timer @@ -969,7 +322,7 @@ to the :class:`Profile` class constructor:: pr = profile.Profile(your_time_func) The resulting profiler will then call ``your_time_func``. Depending on whether -you are using :class:`profile.Profile` or :class:`cProfile.Profile`, +you are using :class:`profile.Profile` or :class:`profiling.tracing.Profile`, ``your_time_func``'s return value will be interpreted differently: :class:`profile.Profile` @@ -988,20 +341,32 @@ you are using :class:`profile.Profile` or :class:`cProfile.Profile`, replacement dispatch method that best handles your timer call, along with the appropriate calibration constant. -:class:`cProfile.Profile` +:class:`profiling.tracing.Profile` ``your_time_func`` should return a single number. If it returns integers, you can also invoke the class constructor with a second argument specifying the real duration of one unit of time. For example, if ``your_integer_time_func`` returns times measured in thousands of seconds, you would construct the :class:`Profile` instance as follows:: - pr = cProfile.Profile(your_integer_time_func, 0.001) + pr = profiling.tracing.Profile(your_integer_time_func, 0.001) - As the :class:`cProfile.Profile` class cannot be calibrated, custom timer - functions should be used with care and should be as fast as possible. For - the best results with a custom timer, it might be necessary to hard-code it - in the C source of the internal :mod:`!_lsprof` module. + As the :class:`profiling.tracing.Profile` class cannot be calibrated, custom + timer functions should be used with care and should be as fast as possible. + For the best results with a custom timer, it might be necessary to hard-code + it in the C source of the internal :mod:`!_lsprof` module. Python 3.3 adds several new functions in :mod:`time` that can be used to make precise measurements of process or wall-clock time. For example, see :func:`time.perf_counter`. + + +.. seealso:: + + :mod:`profiling` + Overview of Python profiling tools. + + :mod:`profiling.tracing` + Recommended replacement for this module. + + :mod:`pstats` + Statistical analysis and formatting for profile data. diff --git a/Doc/library/profiling-sampling-visualization.html b/Doc/library/profiling-sampling-visualization.html new file mode 100644 index 000000000000000..0cbd0f2374deaa0 --- /dev/null +++ b/Doc/library/profiling-sampling-visualization.html @@ -0,0 +1 @@ +
diff --git a/Doc/library/profiling.rst b/Doc/library/profiling.rst new file mode 100644 index 000000000000000..f4ac47826b28ef0 --- /dev/null +++ b/Doc/library/profiling.rst @@ -0,0 +1,270 @@ +.. highlight:: sh + +.. _profiling-module: + +************************************** +:mod:`!profiling` --- Python profilers +************************************** + +.. module:: profiling + :synopsis: Python profiling tools for performance analysis. + +.. versionadded:: 3.15 + +**Source code:** :source:`Lib/profiling/` + +-------------- + +.. index:: + single: statistical profiling + single: profiling, statistical + single: deterministic profiling + single: profiling, deterministic + + +Introduction to profiling +========================= + +A :dfn:`profile` is a set of statistics that describes how often and for how +long various parts of a program execute. These statistics help identify +performance bottlenecks and guide optimization efforts. Python provides two +fundamentally different approaches to collecting this information: statistical +sampling and deterministic tracing. + +The :mod:`!profiling` package organizes Python's built-in profiling tools under +a single namespace. It contains two submodules, each implementing a different +profiling methodology: + +:mod:`profiling.sampling` + A statistical profiler that periodically samples the call stack. Run scripts + directly or attach to running processes by PID. Provides multiple output + formats (flame graphs, heatmaps, Firefox Profiler), GIL analysis, GC tracking, + and multiple profiling modes (wall-clock, CPU, GIL) with virtually no overhead. + +:mod:`profiling.tracing` + A deterministic profiler that traces every function call, return, and + exception event. Provides exact call counts and precise timing information, + capturing every invocation including very fast functions. + +.. note:: + + The profiler modules are designed to provide an execution profile for a + given program, not for benchmarking purposes. For benchmarking, use the + :mod:`timeit` module, which provides reasonably accurate timing + measurements. This distinction is particularly important when comparing + Python code against C code: deterministic profilers introduce overhead for + Python code but not for C-level functions, which can skew comparisons. + + +.. _choosing-a-profiler: + +Choosing a profiler +=================== + +For most performance analysis, use the statistical profiler +(:mod:`profiling.sampling`). It has minimal overhead, works for both development +and production, and provides rich visualization options including flame graphs, +heatmaps, GIL analysis, and more. + +Use the deterministic profiler (:mod:`profiling.tracing`) when you need **exact +call counts** and cannot afford to miss any function calls. Since it instruments +every function call and return, it will capture even very fast functions that +complete between sampling intervals. The tradeoff is higher overhead. + +The following table summarizes the key differences: + ++--------------------+------------------------------+------------------------------+ +| Feature | Statistical sampling | Deterministic | +| | (:mod:`profiling.sampling`) | (:mod:`profiling.tracing`) | ++====================+==============================+==============================+ +| **Overhead** | Virtually none | Moderate | ++--------------------+------------------------------+------------------------------+ +| **Accuracy** | Statistical estimate | Exact call counts | ++--------------------+------------------------------+------------------------------+ +| **Output formats** | pstats, flame graph, heatmap,| pstats | +| | gecko, collapsed | | ++--------------------+------------------------------+------------------------------+ +| **Profiling modes**| Wall-clock, CPU, GIL | Wall-clock | ++--------------------+------------------------------+------------------------------+ +| **Special frames** | GC, native (C extensions) | N/A | ++--------------------+------------------------------+------------------------------+ +| **Attach to PID** | Yes | No | ++--------------------+------------------------------+------------------------------+ + + +When to use statistical sampling +-------------------------------- + +The statistical profiler (:mod:`profiling.sampling`) is recommended for most +performance analysis tasks. Use it the same way you would use +:mod:`profiling.tracing`:: + + python -m profiling.sampling run script.py + +One of the main strengths of the sampling profiler is its variety of output +formats. Beyond traditional pstats tables, it can generate interactive +flame graphs that visualize call hierarchies, line-level source heatmaps that +show exactly where time is spent in your code, and Firefox Profiler output for +timeline-based analysis. + +The profiler also provides insight into Python interpreter behavior that +deterministic profiling cannot capture. Use ``--mode gil`` to identify GIL +contention in multi-threaded code, ``--mode cpu`` to measure actual CPU time +excluding I/O waits, or inspect ```` frames to understand garbage collection +overhead. The ``--native`` option reveals time spent in C extensions, helping +distinguish Python overhead from library performance. + +For multi-threaded applications, the ``-a`` option samples all threads +simultaneously, showing how work is distributed. And for production debugging, +the ``attach`` command connects to any running Python process by PID without +requiring a restart or code changes. + + +When to use deterministic tracing +--------------------------------- + +The deterministic profiler (:mod:`profiling.tracing`) instruments every function +call and return. This approach has higher overhead than sampling, but guarantees +complete coverage of program execution. + +The primary reason to choose deterministic tracing is when you need exact call +counts. Statistical profiling estimates frequency based on sampling, which may +undercount short-lived functions that complete between samples. If you need to +verify that an optimization actually reduced the number of function calls, or +if you want to trace the complete call graph to understand caller-callee +relationships, deterministic tracing is the right choice. + +Deterministic tracing also excels at capturing functions that execute in +microseconds. Such functions may not appear frequently enough in statistical +samples, but deterministic tracing records every invocation regardless of +duration. + + +Quick start +=========== + +This section provides the minimal steps needed to start profiling. For complete +documentation, see the dedicated pages for each profiler. + + +Statistical profiling +--------------------- + +To profile a script, use the :mod:`profiling.sampling` module with the ``run`` +command:: + + python -m profiling.sampling run script.py + python -m profiling.sampling run -m mypackage.module + +This runs the script under the profiler and prints a summary of where time was +spent. For an interactive flame graph:: + + python -m profiling.sampling run --flamegraph script.py + +To profile an already-running process, use the ``attach`` command with the +process ID:: + + python -m profiling.sampling attach 1234 + +For custom settings, specify the sampling interval (in microseconds) and +duration (in seconds):: + + python -m profiling.sampling run -i 50 -d 30 script.py + + +Deterministic profiling +----------------------- + +To profile a script from the command line:: + + python -m profiling.tracing script.py + +To profile a piece of code programmatically: + +.. code-block:: python + + import profiling.tracing + profiling.tracing.run('my_function()') + +This executes the given code under the profiler and prints a summary showing +exact function call counts and timing. + + +.. _profile-output: + +Understanding profile output +============================ + +Both profilers collect function-level statistics, though they present them in +different formats. The sampling profiler offers multiple visualizations +(flame graphs, heatmaps, Firefox Profiler, pstats tables), while the +deterministic profiler produces pstats-compatible output. Regardless of format, +the underlying concepts are the same. + +Key profiling concepts: + +**Direct time** (also called *self time* or *tottime*) + Time spent executing code in the function itself, excluding time spent in + functions it called. High direct time indicates the function contains + expensive operations. + +**Cumulative time** (also called *total time* or *cumtime*) + Time spent in the function and all functions it called. This measures the + total cost of calling a function, including its entire call subtree. + +**Call count** (also called *ncalls* or *samples*) + How many times the function was called (deterministic) or sampled + (statistical). In deterministic profiling, this is exact. In statistical + profiling, it represents the number of times the function appeared in a + stack sample. + +**Primitive calls** + Calls that are not induced by recursion. When a function recurses, the total + call count includes recursive invocations, but primitive calls counts only + the initial entry. Displayed as ``total/primitive`` (for example, ``3/1`` + means three total calls, one primitive). + +**Caller/Callee relationships** + Which functions called a given function (callers) and which functions it + called (callees). Flame graphs visualize this as nested rectangles; pstats + can display it via the :meth:`~pstats.Stats.print_callers` and + :meth:`~pstats.Stats.print_callees` methods. + + +Legacy compatibility +==================== + +For backward compatibility, the ``cProfile`` module remains available as an +alias to :mod:`profiling.tracing`. Existing code using ``import cProfile`` will +continue to work without modification in all future Python versions. + +.. deprecated:: 3.15 + + The pure Python :mod:`profile` module is deprecated and will be removed in + Python 3.17. Use :mod:`profiling.tracing` (or its alias ``cProfile``) + instead. See :mod:`profile` for migration guidance. + + +.. seealso:: + + :mod:`profiling.sampling` + Statistical sampling profiler with flame graphs, heatmaps, and GIL analysis. + Recommended for most users. + + :mod:`profiling.tracing` + Deterministic tracing profiler for exact call counts. + + :mod:`pstats` + Statistics analysis and formatting for profile data. + + :mod:`timeit` + Module for measuring execution time of small code snippets. + + +.. rubric:: Submodules + +.. toctree:: + :maxdepth: 1 + + profiling.tracing.rst + profiling.sampling.rst diff --git a/Doc/library/profiling.sampling.rst b/Doc/library/profiling.sampling.rst new file mode 100644 index 000000000000000..aeb1a429b58515a --- /dev/null +++ b/Doc/library/profiling.sampling.rst @@ -0,0 +1,1735 @@ +.. highlight:: sh + +.. _profiling-sampling: + +*************************************************** +:mod:`!profiling.sampling` --- Statistical profiler +*************************************************** + +.. module:: profiling.sampling + :synopsis: Statistical sampling profiler for Python processes. + +.. versionadded:: 3.15 + +**Source code:** :source:`Lib/profiling/sampling/` + +.. program:: profiling.sampling + +-------------- + +.. image:: ../../Lib/profiling/sampling/_assets/tachyon-logo.png + :alt: Tachyon logo + :align: center + :width: 300px + +The :mod:`!profiling.sampling` module, named **Tachyon**, provides statistical +profiling of Python programs through periodic stack sampling. Tachyon can +run scripts directly or attach to any running Python process without requiring +code changes or restarts. Because sampling occurs externally to the target +process, overhead is virtually zero, making Tachyon suitable for both +development and production environments. + + +What is statistical profiling? +============================== + +Statistical profiling builds a picture of program behavior by periodically +capturing snapshots of the call stack. Rather than instrumenting every function +call and return as deterministic profilers do, Tachyon reads the call stack at +regular intervals to record what code is currently running. + +This approach rests on a simple principle: functions that consume significant +CPU time will appear frequently in the collected samples. By gathering thousands +of samples over a profiling session, Tachyon constructs an accurate statistical +estimate of where time is spent. The more samples collected, the +more precise this estimate becomes. + +.. only:: html + + The following interactive visualization demonstrates how sampling profiling + works. Press **Play** to watch a Python program execute, and observe how the + profiler periodically captures snapshots of the call stack. Adjust the + **sample interval** to see how sampling frequency affects the results. + + .. raw:: html + :file: profiling-sampling-visualization.html + +.. only:: not html + + .. note:: + + An interactive visualization of sampling profiling is available in the + HTML version of this documentation. + + +How time is estimated +--------------------- + +The time values shown in Tachyon's output are **estimates derived from sample +counts**, not direct measurements. Tachyon counts how many times each function +appears in the collected samples, then multiplies by the sampling interval to +estimate time. + +For example, with a 10 kHz sampling rate over a 10-second profile, +Tachyon collects approximately 100,000 samples. If a function appears in 5,000 +samples (5% of total), Tachyon estimates it consumed 5% of the 10-second +duration, or about 500 milliseconds. This is a statistical estimate, not a +precise measurement. + +The accuracy of these estimates depends on sample count. With 100,000 samples, +a function showing 5% has a margin of error of roughly ±0.5%. With only 1,000 +samples, the same 5% measurement could actually represent anywhere from 3% to +7% of real time. + +This is why longer profiling durations and shorter sampling intervals produce +more reliable results---they collect more samples. For most performance +analysis, the default settings provide sufficient accuracy to identify +bottlenecks and guide optimization efforts. + +Because sampling is statistical, results will vary slightly between runs. A +function showing 12% in one run might show 11% or 13% in the next. This is +normal and expected. Focus on the overall pattern rather than exact percentages, +and don't worry about small variations between runs. + + +When to use a different approach +-------------------------------- + +Statistical sampling is not ideal for every situation. + +For very short scripts that complete in under one second, the profiler may not +collect enough samples for reliable results. Use :mod:`profiling.tracing` +instead, or run the script in a loop to extend profiling time. + +When you need exact call counts, sampling cannot provide them. Sampling +estimates frequency from snapshots, so if you need to know precisely how many +times a function was called, use :mod:`profiling.tracing`. + +When comparing two implementations where the difference might be only 1-2%, +sampling noise can obscure real differences. Use :mod:`timeit` for +micro-benchmarks or :mod:`profiling.tracing` for precise measurements. + + +The key difference from :mod:`profiling.tracing` is how measurement happens. +A tracing profiler instruments your code, recording every function call and +return. This provides exact call counts and precise timing but adds overhead +to every function call. A sampling profiler, by contrast, observes the program +from outside at fixed intervals without modifying its execution. Think of the +difference like this: tracing is like having someone follow you and write down +every step you take, while sampling is like taking photographs every second +and inferring your path from those snapshots. + +This external observation model is what makes sampling profiling practical for +production use. The profiled program runs at full speed because there is no +instrumentation code running inside it, and the target process is never stopped +or paused during sampling---Tachyon reads the call stack directly from the +process's memory while it continues to run. You can attach to a live server, +collect data, and detach without the application ever knowing it was observed. +The trade-off is that very short-lived functions may be missed if they happen +to complete between samples. + +Statistical profiling excels at answering the question, "Where is my program +spending time?" It reveals hotspots and bottlenecks in production code where +deterministic profiling overhead would be unacceptable. For exact call counts +and complete call graphs, use :mod:`profiling.tracing` instead. + + +Quick examples +============== + +Profile a script and see the results immediately:: + + python -m profiling.sampling run script.py + +Profile a module with arguments:: + + python -m profiling.sampling run -m mypackage.module arg1 arg2 + +Generate an interactive flame graph:: + + python -m profiling.sampling run --flamegraph -o profile.html script.py + +Attach to a running process by PID:: + + python -m profiling.sampling attach 12345 + +Print a single snapshot of a running process's stack:: + + python -m profiling.sampling dump 12345 + +Use live mode for real-time monitoring (press ``q`` to quit):: + + python -m profiling.sampling run --live script.py + +Profile for 60 seconds with a faster sampling rate:: + + python -m profiling.sampling run -d 60 -r 20khz script.py + +Generate a line-by-line heatmap:: + + python -m profiling.sampling run --heatmap script.py + +Enable opcode-level profiling to see which bytecode instructions are executing:: + + python -m profiling.sampling run --opcodes --flamegraph script.py + + +Commands +======== + +Tachyon operates through several subcommands. ``run`` and ``attach`` collect +samples over time; ``dump`` captures a single snapshot; ``replay`` converts +binary profiles to other formats. + + +The ``run`` command +------------------- + +The ``run`` command launches a Python script or module and profiles it from +startup:: + + python -m profiling.sampling run script.py + python -m profiling.sampling run -m mypackage.module + +When profiling a script, the profiler starts the target in a subprocess, waits +for it to initialize, then begins collecting samples. The ``-m`` flag +indicates that the target should be run as a module (equivalent to +``python -m``). Arguments after the target are passed through to the +profiled program:: + + python -m profiling.sampling run script.py --config settings.yaml + + +The ``attach`` command +---------------------- + +The ``attach`` command connects to an already-running Python process by its +process ID:: + + python -m profiling.sampling attach 12345 + +This command is particularly valuable for investigating performance issues in +production systems. The target process requires no modification and need not +be restarted. The profiler attaches, collects samples for the specified +duration, then detaches and produces output. + +:: + + python -m profiling.sampling attach --live 12345 + python -m profiling.sampling attach --flamegraph -d 30 -o profile.html 12345 + +On most systems, attaching to another process requires appropriate permissions. +See :ref:`profiling-permissions` for platform-specific requirements. + + +.. _dump-command: + +The ``dump`` command +-------------------- + +The ``dump`` command prints a single snapshot of a running process's Python +stack and exits, similar to a traceback:: + + python -m profiling.sampling dump 12345 + +Unlike ``attach``, ``dump`` does not run a sampling loop: it reads the +stack once. This is useful for investigating hung or unresponsive +processes, or for answering "what is this process doing right now?". + +The output mirrors a traceback (most recent call last) and annotates each +thread with its current state (main thread, has GIL, on CPU, waiting for +GIL, has exception, or idle): + +.. code-block:: text + + Stack dump for PID 12345, thread 140735 (main thread, has GIL, on CPU; most recent call last): + File "server.py", line 28, in serve + await handle_request(req) + File "handler.py", line 91, in handle_request + result = expensive_call(req) + +When the target's source files are readable, ``dump`` prints the source +line for each frame and highlights the executing expression. + +Like ``attach``, ``dump`` requires permission to read the target process's +memory. See :ref:`profiling-permissions`. + +The ``dump`` command supports the following options: + +``-a``, ``--all-threads`` + Dump every thread in the target process. Without this flag only the main + thread is shown. + +``--native`` + Include synthetic ```` frames marking transitions into C + extensions or other non-Python code. + +``--no-gc`` + Hide the synthetic ```` frames that mark active garbage collection. + +``--opcodes`` + Annotate each frame with the bytecode opcode the thread is currently + executing (for example, ``opcode=CALL_KW``). Useful for + instruction-level investigation, including identifying specializations + chosen by the adaptive interpreter. + +``--async-aware`` + Reconstruct stacks across ``await`` boundaries. ``dump`` walks the task + graph and emits one section per task, with ```` markers separating + coroutines awaiting each other. + +``--async-mode {running,all}`` + Controls which tasks are included when ``--async-aware`` is enabled. + ``running`` shows only the task currently executing on each thread; + ``all`` (the default for ``dump``) also includes tasks suspended on a + wait. ``attach``'s default for this flag is ``running``; ``dump`` + defaults to ``all`` because a single snapshot is most useful when it + shows the full task graph. + +``--blocking`` + Pause every thread in the target while reading its stack and resume + them after. Guarantees a fully consistent snapshot at the cost of + briefly stopping the target. Without it, ``dump`` reads memory while + the target keeps running, which is faster but can occasionally produce + a torn stack. + + +.. _replay-command: + +The ``replay`` command +---------------------- + +The ``replay`` command converts binary profile files to other output formats:: + + python -m profiling.sampling replay profile.bin + python -m profiling.sampling replay --flamegraph -o profile.html profile.bin + +This command is useful when you have captured profiling data in binary format +and want to analyze it later or convert it to a visualization format. Binary +profiles can be replayed multiple times to different formats without +re-profiling. + +:: + + # Convert binary to pstats (default, prints to stdout) + python -m profiling.sampling replay profile.bin + + # Convert binary to flame graph + python -m profiling.sampling replay --flamegraph -o output.html profile.bin + + # Convert binary to gecko format for Firefox Profiler + python -m profiling.sampling replay --gecko -o profile.json profile.bin + + # Convert binary to heatmap + python -m profiling.sampling replay --heatmap -o my_heatmap profile.bin + + +Profiling in production +----------------------- + +The sampling profiler is designed for production use. It imposes no measurable +overhead on the target process because it reads memory externally rather than +instrumenting code. The target application continues running at full speed and +is unaware it is being profiled. + +When profiling production systems, keep these guidelines in mind: + +Start with shorter durations (10-30 seconds) to get quick results, then extend +if you need more statistical accuracy. By default, profiling runs until the +target process completes, which is usually sufficient to identify major hotspots. + +If possible, profile during representative load rather than peak traffic. +Profiles collected during normal operation are easier to interpret than those +collected during unusual spikes. + +The profiler itself consumes some CPU on the machine where it runs (not on the +target process). On the same machine, this is typically negligible. When +profiling remote processes, network latency does not affect the target. + +Results from production may differ from development due to different data +sizes, concurrent load, or caching effects. This is expected and is often +exactly what you want to capture. + + +.. _profiling-permissions: + +Platform requirements +--------------------- + +The profiler reads the target process's memory to capture stack traces. This +requires elevated permissions on most operating systems. + +**Linux** + +On Linux, the profiler uses ``ptrace`` or ``process_vm_readv`` to read the +target process's memory. This typically requires one of: + +- Running as root +- Having the ``CAP_SYS_PTRACE`` capability +- Adjusting the Yama ptrace scope: ``/proc/sys/kernel/yama/ptrace_scope`` + +The default ptrace_scope of 1 restricts ptrace to parent processes only. To +allow attaching to any process owned by the same user, set it to 0:: + + echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope + +**macOS** + +On macOS, the profiler uses ``task_for_pid()`` to access the target process. +This requires one of: + +- Running as root +- The profiler binary having the ``com.apple.security.cs.debugger`` entitlement +- System Integrity Protection (SIP) being disabled (not recommended) + +**Windows** + +On Windows, the profiler requires administrative privileges or the +``SeDebugPrivilege`` privilege to read another process's memory. + + +Version compatibility +--------------------- + +The profiler and target process must run the same Python minor version (for +example, both Python 3.15). Attaching from Python 3.14 to a Python 3.15 process +is not supported. + +Additional restrictions apply to pre-release Python versions: if either the +profiler or target is running a pre-release (alpha, beta, or release candidate), +both must run the exact same version. + +On free-threaded Python builds, the profiler cannot attach from a free-threaded +build to a standard build, or vice versa. + + +Sampling configuration +====================== + +Before exploring the various output formats and visualization options, it is +important to understand how to configure the sampling process itself. The +profiler offers several options that control how frequently samples are +collected, how long profiling runs, which threads are observed, and what +additional context is captured in each sample. + +The default configuration works well for most use cases: + +.. list-table:: + :header-rows: 1 + :widths: 25 75 + + * - Option + - Default + * - Default for ``--sampling-rate`` / ``-r`` + - 1 kHz + * - Default for ``--duration`` / ``-d`` + - Run to completion + * - Default for ``--all-threads`` / ``-a`` + - Main thread only + * - Default for ``--native`` + - No ```` frames (C code time attributed to caller) + * - Default for ``--no-gc`` + - ```` frames included when garbage collection is active + * - Default for ``--mode`` + - Wall-clock mode (all samples recorded) + * - Default for ``--realtime-stats`` + - Disabled + * - Default for ``--subprocesses`` + - Disabled + * - Default for ``--blocking`` + - Disabled (non-blocking sampling) + + +Sampling rate and duration +-------------------------- + +The two most fundamental parameters are the sampling rate and duration. +Together, these determine how many samples will be collected during a profiling +session. + +The :option:`--sampling-rate` option (:option:`-r`) sets how frequently samples +are collected. The default is 1 kHz (1,000 samples per second):: + + python -m profiling.sampling run -r 20khz script.py + +Higher rates capture more samples and provide finer-grained data at the +cost of slightly higher profiler CPU usage. Lower rates reduce profiler +overhead but may miss short-lived functions. For most applications, the +default rate provides a good balance between accuracy and overhead. + +The :option:`--duration` option (:option:`-d`) sets how long to profile in seconds. By +default, profiling continues until the target process exits or is interrupted:: + + python -m profiling.sampling run -d 60 script.py + +Specifying a duration is useful when attaching to long-running processes or when +you want to limit profiling to a specific time window. When profiling a script, +the default behavior of running to completion is usually what you want. + + +Thread selection +---------------- + +Python programs often use multiple threads, whether explicitly through the +:mod:`threading` module or implicitly through libraries that manage thread +pools. + +By default, the profiler samples only the main thread. The :option:`--all-threads` +option (:option:`-a`) enables sampling of all threads in the process:: + + python -m profiling.sampling run -a script.py + +Multi-thread profiling reveals how work is distributed across threads and can +identify threads that are blocked or starved. Each thread's samples are +combined in the output, with the ability to filter by thread in some formats. +This option is particularly useful when investigating concurrency issues or +when work is distributed across a thread pool. + + +.. _blocking-mode: + +Blocking mode +------------- + +By default, Tachyon reads the target process's memory without stopping it. +This non-blocking approach is ideal for most profiling scenarios because it +imposes virtually zero overhead on the target application: the profiled +program runs at full speed and is unaware it is being observed. + +However, non-blocking sampling can occasionally produce incomplete or +inconsistent stack traces in applications with many generators or coroutines +that rapidly switch between yield points, or in programs with very fast-changing +call stacks where functions enter and exit between the start and end of a single +stack read, resulting in reconstructed stacks that mix frames from different +execution states or that never actually existed. + +For these cases, the :option:`--blocking` option stops the target process during +each sample:: + + python -m profiling.sampling run --blocking script.py + python -m profiling.sampling attach --blocking 12345 + +When blocking mode is enabled, the profiler suspends the target process, +reads its stack, then resumes it. This guarantees that each captured stack +represents a real, consistent snapshot of what the process was doing at that +instant. The trade-off is that the target process runs slower because it is +repeatedly paused. + +.. warning:: + + Do not use very high sample rates (low ``--interval`` values) with blocking + mode. Suspending and resuming a process takes time, and if the sampling + interval is too short, the target will spend more time stopped than running. + For blocking mode, intervals of 1000 microseconds (1 millisecond) or higher + are recommended. The default 100 microsecond interval may cause noticeable + slowdown in the target application. + +Use blocking mode only when you observe inconsistent stacks in your profiles, +particularly with generator-heavy or coroutine-heavy code. For most +applications, the default non-blocking mode provides accurate results with +zero impact on the target process. + + +Special frames +-------------- + +The profiler can inject artificial frames into the captured stacks to provide +additional context about what the interpreter is doing at the moment each +sample is taken. These synthetic frames help distinguish different types of +execution that would otherwise be invisible. + +The :option:`--native` option adds ```` frames to indicate when Python has +called into C code (extension modules, built-in functions, or the interpreter +itself):: + + python -m profiling.sampling run --native script.py + +These frames help distinguish time spent in Python code versus time spent in +native libraries. Without this option, native code execution appears as time +in the Python function that made the call. This is useful when optimizing +code that makes heavy use of C extensions like NumPy or database drivers. + +By default, the profiler includes ```` frames when garbage collection is +active. The :option:`--no-gc` option suppresses these frames:: + + python -m profiling.sampling run --no-gc script.py + +GC frames help identify programs where garbage collection consumes significant +time, which may indicate memory allocation patterns worth optimizing. If you +see substantial time in ```` frames, consider investigating object +allocation rates or using object pooling. + + +Opcode-aware profiling +---------------------- + +The :option:`--opcodes` option enables instruction-level profiling that captures +which Python bytecode instructions are executing at each sample:: + + python -m profiling.sampling run --opcodes --flamegraph script.py + +This feature provides visibility into Python's bytecode execution, including +adaptive specialization optimizations. When a generic instruction like +``LOAD_ATTR`` is specialized at runtime into a more efficient variant like +``LOAD_ATTR_INSTANCE_VALUE``, the profiler shows both the specialized name +and the base instruction. + +Opcode information appears in several output formats: + +- **Flame graphs**: Hovering over a frame displays a tooltip with a bytecode + instruction breakdown, showing which opcodes consumed time in that function +- **Heatmap**: Expandable bytecode panels per source line show instruction + breakdown with specialization percentages +- **Live mode**: An opcode panel shows instruction-level statistics for the + selected function, accessible via keyboard navigation +- **Gecko format**: Opcode transitions are emitted as interval markers in the + Firefox Profiler timeline + +This level of detail is particularly useful for: + +- Understanding the performance impact of Python's adaptive specialization +- Identifying hot bytecode instructions that might benefit from optimization +- Analyzing the effectiveness of different code patterns at the instruction level +- Debugging performance issues that occur at the bytecode level + +The :option:`--opcodes` option is compatible with :option:`--live`, :option:`--flamegraph`, +:option:`--heatmap`, and :option:`--gecko` formats. It requires additional memory to store +opcode information and may slightly reduce sampling performance, but provides +unprecedented visibility into Python's execution model. + + +Real-time statistics +-------------------- + +The :option:`--realtime-stats` option displays sampling rate statistics during +profiling:: + + python -m profiling.sampling run --realtime-stats script.py + +This shows the actual achieved sampling rate, which may be lower than requested +if the profiler cannot keep up. The statistics help verify that profiling is +working correctly and that sufficient samples are being collected. See +:ref:`sampling-efficiency` for details on interpreting these metrics. + + +Subprocess profiling +-------------------- + +The :option:`--subprocesses` option enables automatic profiling of subprocesses +spawned by the target:: + + python -m profiling.sampling run --subprocesses script.py + python -m profiling.sampling attach --subprocesses 12345 + +When enabled, the profiler monitors the target process for child process +creation. When a new Python child process is detected, a separate profiler +instance is automatically spawned to profile it. This is useful for +applications that use :mod:`multiprocessing`, :mod:`subprocess`, +:mod:`concurrent.futures` with :class:`~concurrent.futures.ProcessPoolExecutor`, +or other process spawning mechanisms. + +.. code-block:: python + :caption: worker_pool.py + + from concurrent.futures import ProcessPoolExecutor + import math + + def compute_factorial(n): + total = 0 + for i in range(50): + total += math.factorial(n) + return total + + if __name__ == "__main__": + numbers = [5000 + i * 100 for i in range(50)] + with ProcessPoolExecutor(max_workers=4) as executor: + results = list(executor.map(compute_factorial, numbers)) + print(f"Computed {len(results)} factorials") + +:: + + python -m profiling.sampling run --subprocesses --flamegraph worker_pool.py + +This produces separate flame graphs for the main process and each worker +process: ``flamegraph_.html``, ``flamegraph_.html``, +and so on. + +Each subprocess receives its own output file. The filename is derived from +the specified output path (or the default) with the subprocess's process ID +appended: + +- If you specify ``-o profile.html``, subprocesses produce ``profile_12345.html``, + ``profile_12346.html``, and so on +- With default output, subprocesses produce files like ``flamegraph_12345.html`` + or directories like ``heatmap_12345`` +- For pstats format (which defaults to stdout), subprocesses produce files like + ``profile_12345.pstats`` + +The subprocess profilers inherit most sampling options from the parent (sampling +rate, duration, thread selection, native frames, GC frames, async-aware mode, +and output format). All Python descendant processes are profiled recursively, +including grandchildren and further descendants. + +Subprocess detection works by periodically scanning for new descendants of +the target process and checking whether each new process is a Python process +by probing the process memory for Python runtime structures. Non-Python +subprocesses (such as shell commands or external tools) are ignored. + +There is a limit of 100 concurrent subprocess profilers to prevent resource +exhaustion in programs that spawn many processes. If this limit is reached, +additional subprocesses are not profiled and a warning is printed. + +The :option:`--subprocesses` option is incompatible with :option:`--live` mode +because live mode uses an interactive terminal interface that cannot +accommodate multiple concurrent profiler displays. + + +.. _sampling-efficiency: + +Sampling efficiency +------------------- + +Sampling efficiency metrics help assess the quality of the collected data. +These metrics appear in the profiler's terminal output and in the flame graph +sidebar. + +**Sampling efficiency** is the percentage of sample attempts that succeeded. +Each sample attempt reads the target process's call stack from memory. An +attempt can fail if the process is in an inconsistent state at the moment of +reading, such as during a context switch or while the interpreter is updating +its internal structures. A low efficiency may indicate that the profiler could +not keep up with the requested sampling rate, often due to system load or an +overly aggressive interval setting. + +**Missed samples** is the percentage of expected samples that were not +collected. Based on the configured interval and duration, the profiler expects +to collect a certain number of samples. Some samples may be missed if the +profiler falls behind schedule, for example when the system is under heavy +load. A small percentage of missed samples is normal and does not significantly +affect the statistical accuracy of the profile. + +Both metrics are informational. Even with some failed attempts or missed +samples, the profile remains statistically valid as long as enough samples +were collected. The profiler reports the actual number of samples captured, +which you can use to judge whether the data is sufficient for your analysis. + + +Profiling modes +=============== + +The sampling profiler supports four modes that control which samples are +recorded. The mode determines what the profile measures: total elapsed time, +CPU execution time, time spent holding the global interpreter lock, or +exception handling. + + +Wall-clock mode +--------------- + +Wall-clock mode (:option:`--mode`\ ``=wall``) captures all samples regardless of what the +thread is doing. This is the default mode and provides a complete picture of +where time passes during program execution:: + + python -m profiling.sampling run --mode=wall script.py + +In wall-clock mode, samples are recorded whether the thread is actively +executing Python code, waiting for I/O, blocked on a lock, or sleeping. +This makes wall-clock profiling ideal for understanding the overall time +distribution in your program, including time spent waiting. + +If your program spends significant time in I/O operations, network calls, or +sleep, wall-clock mode will show these waits as time attributed to the calling +function. This is often exactly what you want when optimizing end-to-end +latency. + + +CPU mode +-------- + +CPU mode (:option:`--mode`\ ``=cpu``) records samples only when the thread is actually +executing on a CPU core:: + + python -m profiling.sampling run --mode=cpu script.py + +Samples taken while the thread is sleeping, blocked on I/O, or waiting for +a lock are discarded. The resulting profile shows where CPU cycles are consumed, +filtering out idle time. + +CPU mode is useful when you want to focus on computational hotspots without +being distracted by I/O waits. If your program alternates between computation +and network calls, CPU mode reveals which computational sections are most +expensive. + + +Comparing wall-clock and CPU profiles +------------------------------------- + +Running both wall-clock and CPU mode profiles can reveal whether a function's +time is spent computing or waiting. + +If a function appears prominently in both profiles, it is a true computational +hotspot---actively using the CPU. Optimization should focus on algorithmic +improvements or more efficient code. + +If a function is high in wall-clock mode but low or absent in CPU mode, it is +I/O-bound or waiting. The function spends most of its time waiting for network, +disk, locks, or sleep. CPU optimization won't help here; consider async I/O, +connection pooling, or reducing wait time instead. + +.. code-block:: python + + import time + + def do_sleep(): + time.sleep(2) + + def do_compute(): + sum(i**2 for i in range(1000000)) + + if __name__ == "__main__": + do_sleep() + do_compute() + +:: + + python -m profiling.sampling run --mode=wall script.py # do_sleep ~98%, do_compute ~1% + python -m profiling.sampling run --mode=cpu script.py # do_sleep absent, do_compute dominates + + +GIL mode +-------- + +GIL mode (:option:`--mode`\ ``=gil``) records samples only when the thread holds Python's +global interpreter lock:: + + python -m profiling.sampling run --mode=gil script.py + +The GIL is held only while executing Python bytecode. When Python calls into +C extensions, performs I/O operations, or executes native code, the GIL is +typically released. This means GIL mode effectively measures time spent +running Python code specifically, filtering out time in native libraries. + +In multi-threaded programs, GIL mode reveals which code is preventing other +threads from running Python bytecode. Since only one thread can hold the GIL +at a time, functions that appear frequently in GIL mode profiles are +monopolizing the interpreter. + +GIL mode helps answer questions like "which functions are monopolizing the +GIL?" and "why are my other threads starving?" It can also be useful in +single-threaded programs to distinguish Python execution time from time spent +in C extensions or I/O. + +.. code-block:: python + + import hashlib + + def hash_work(): + # C extension - releases GIL during computation + for _ in range(200): + hashlib.sha256(b"data" * 250000).hexdigest() + + def python_work(): + # Pure Python - holds GIL during computation + for _ in range(3): + sum(i**2 for i in range(1000000)) + + if __name__ == "__main__": + hash_work() + python_work() + +:: + + python -m profiling.sampling run --mode=cpu script.py # hash_work ~42%, python_work ~38% + python -m profiling.sampling run --mode=gil script.py # hash_work ~5%, python_work ~60% + + +Exception mode +-------------- + +Exception mode (``--mode=exception``) records samples only when a thread has +an active exception:: + + python -m profiling.sampling run --mode=exception script.py + +Samples are recorded in two situations: when an exception is being propagated +up the call stack (after ``raise`` but before being caught), or when code is +executing inside an ``except`` block where exception information is still +present in the thread state. + +The following example illustrates which code regions are captured: + +.. code-block:: python + + def example(): + try: + raise ValueError("error") # Captured: exception being raised + except ValueError: + process_error() # Captured: inside except block + finally: + cleanup() # NOT captured: exception already handled + + def example_propagating(): + try: + try: + raise ValueError("error") + finally: + cleanup() # Captured: exception propagating through + except ValueError: + pass + + def example_no_exception(): + try: + do_work() + finally: + cleanup() # NOT captured: no exception involved + +Note that ``finally`` blocks are only captured when an exception is actively +propagating through them. Once an ``except`` block finishes executing, Python +clears the exception information before running any subsequent ``finally`` +block. Similarly, ``finally`` blocks that run during normal execution (when no +exception was raised) are not captured because no exception state is present. + +This mode is useful for understanding where your program spends time handling +errors. Exception handling can be a significant source of overhead in code +that uses exceptions for flow control (such as ``StopIteration`` in iterators) +or in applications that process many error conditions (such as network servers +handling connection failures). + +Exception mode helps answer questions like "how much time is spent handling +exceptions?" and "which exception handlers are the most expensive?" It can +reveal hidden performance costs in code that catches and processes many +exceptions, even when those exceptions are handled gracefully. For example, +if a parsing library uses exceptions internally to signal format errors, this +mode will capture time spent in those handlers even if the calling code never +sees the exceptions. + + +Output formats +============== + +The profiler produces output in several formats, each suited to different +analysis workflows. The format is selected with a command-line flag, and +output goes to stdout, a file, or a directory depending on the format. + + +pstats format +------------- + +The pstats format (:option:`--pstats`) produces a text table similar to what +deterministic profilers generate. This is the default output format:: + + python -m profiling.sampling run script.py + python -m profiling.sampling run --pstats script.py + +.. figure:: tachyon-pstats.png + :alt: Tachyon pstats terminal output + :align: center + :width: 100% + + The pstats format displays profiling results in a color-coded table showing + function hotspots, sample counts, and timing estimates. + +Output appears on stdout by default:: + + Profile Stats (Mode: wall): + nsamples sample% tottime (ms) cumul% cumtime (ms) filename:lineno(function) + 234/892 11.7% 234.00 44.6% 892.00 server.py:145(handle_request) + 156/156 7.8% 156.00 7.8% 156.00 :0(socket.recv) + 98/421 4.9% 98.00 21.1% 421.00 parser.py:67(parse_message) + +The columns show sampling counts and estimated times: + +- **nsamples**: Displayed as ``direct/cumulative`` (for example, ``10/50``). + Direct samples are when the function was at the top of the stack, actively + executing. Cumulative samples are when the function appeared anywhere on the + stack, including when it was waiting for functions it called. If a function + shows ``10/50``, it was directly executing in 10 samples and was on the call + stack in 50 samples total. + +- **sample%** and **cumul%**: Percentages of total samples for direct and + cumulative counts respectively. + +- **tottime** and **cumtime**: Estimated wall-clock time based on sample counts + and the profiling duration. Time units are selected automatically based on + the magnitude: seconds for large values, milliseconds for moderate values, + or microseconds for small values. + +The output includes a legend explaining each column and a summary of +interesting functions that highlights: + +- **Hot spots**: Functions with high direct/cumulative sample ratio (ratio + close to 1.0). These functions spend most of their time executing their own + code rather than waiting for callees. High ratios indicate where CPU time + is actually consumed. + +- **Indirect calls**: Functions with large differences between cumulative and + direct samples. These are orchestration functions that delegate work to + other functions. They appear frequently on the stack but rarely at the top. + +- **Call magnification**: Functions where cumulative samples far exceed direct + samples (high cumulative/direct multiplier). These are frequently-nested + functions that appear deep in many call chains. + +Use :option:`--no-summary` to suppress both the legend and summary sections. + +To save pstats output to a binary file instead of stdout:: + + python -m profiling.sampling run -o profile.pstats script.py + +The pstats format supports several options for controlling the display. +The :option:`--sort` option determines the column used for ordering results:: + + python -m profiling.sampling run --sort=tottime script.py + python -m profiling.sampling run --sort=cumtime script.py + python -m profiling.sampling run --sort=nsamples script.py + +The :option:`--limit` option restricts output to the top N entries:: + + python -m profiling.sampling run --limit=30 script.py + +The :option:`--no-summary` option suppresses the header summary that precedes the +statistics table. + + +Collapsed stacks format +----------------------- + +Collapsed stacks format (:option:`--collapsed`) produces one line per unique call +stack, with a count of how many times that stack was sampled:: + + python -m profiling.sampling run --collapsed script.py + +The output looks like: + +.. code-block:: text + + main;process_data;parse_json;decode_utf8 42 + main;process_data;parse_json 156 + main;handle_request;send_response 89 + +Each line contains semicolon-separated function names representing the call +stack from bottom to top, followed by a space and the sample count. This +format is designed for compatibility with external flame graph tools, +particularly Brendan Gregg's ``flamegraph.pl`` script. + +To generate a flame graph from collapsed stacks:: + + python -m profiling.sampling run --collapsed script.py > stacks.txt + flamegraph.pl stacks.txt > profile.svg + +The resulting SVG can be viewed in any web browser and provides an interactive +visualization where you can click to zoom into specific call paths. + + +Flame graph format +------------------ + +Flame graph format (:option:`--flamegraph`) produces a self-contained HTML file with +an interactive flame graph visualization:: + + python -m profiling.sampling run --flamegraph script.py + python -m profiling.sampling run --flamegraph -o profile.html script.py + +.. figure:: tachyon-flamegraph.png + :alt: Tachyon interactive flame graph + :align: center + :width: 100% + + The flame graph visualization shows call stacks as nested rectangles, with + width proportional to time spent. The sidebar displays runtime statistics, + GIL metrics, and hotspot functions. + +.. only:: html + + `Try the interactive example <../_static/tachyon-example-flamegraph.html>`__! + +If no output file is specified, the profiler generates a filename based on +the process ID (for example, ``flamegraph.12345.html``). + +The generated HTML file requires no external dependencies and can be opened +directly in a web browser. The visualization displays call stacks as nested +rectangles, with width proportional to time spent. Hovering over a rectangle +shows details about that function including source code context, and clicking +zooms into that portion of the call tree. + +The flame graph interface includes: + +- A sidebar showing profile summary, thread statistics, sampling efficiency + metrics (see :ref:`sampling-efficiency`), and top hotspot functions +- Search functionality supporting both function name matching and + ``file.py:42`` line patterns +- Per-thread filtering via dropdown +- Dark/light theme toggle (preference saved across sessions) +- SVG export for saving the current view + +The thread statistics section shows runtime behavior metrics: + +- **GIL Held**: percentage of samples where a thread held the global interpreter + lock (actively running Python code) +- **GIL Released**: percentage of samples where no thread held the GIL +- **Waiting GIL**: percentage of samples where a thread was waiting to acquire + the GIL +- **GC**: percentage of samples during garbage collection + +These statistics help identify GIL contention and understand how time is +distributed between Python execution, native code, and waiting. + +Flame graphs are particularly effective for identifying deep call stacks and +understanding the hierarchical structure of time consumption. Wide rectangles +at the top indicate functions that consume significant time either directly +or through their callees. + + +Differential flame graphs +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Differential flame graphs compare two profiling runs to highlight where +performance changed. This helps identify regressions introduced by code changes +and validate that optimizations achieved their intended effect:: + + # Capture baseline profile + python -m profiling.sampling run --binary -o baseline.bin script.py + + # After modifying code, generate differential flamegraph + python -m profiling.sampling run --diff-flamegraph baseline.bin -o diff.html script.py + +The visualization draws the current profile with frame widths showing current +time consumption, then applies color to indicate how each function changed +relative to the baseline. + +**Color coding**: + +- **Red**: Functions consuming more time (regressions). Lighter shades indicate + modest increases, while darker shades show severe regressions. + +- **Blue**: Functions consuming less time (improvements). Lighter shades for + modest reductions, darker shades for significant speedups. + +- **Gray**: Minimal or no change. + +- **Purple**: New functions not present in the baseline. + +Frame colors indicate changes in **direct time** (time when the function was at +the top of the stack, actively executing), not cumulative time including callees. +Hovering over a frame shows comparison details including baseline time, current +time, and the percentage change. + +Some call paths may disappear entirely between profiles. These are called +**elided stacks** and occur when optimizations eliminate code paths or certain +branches stop executing. If elided stacks are present, an elided toggle appears +allowing you to switch between the main differential view and an elided-only +view that shows just the removed paths (colored purple). + + +Gecko format +------------ + +Gecko format (:option:`--gecko`) produces JSON output compatible with the Firefox +Profiler:: + + python -m profiling.sampling run --gecko script.py + python -m profiling.sampling run --gecko -o profile.json script.py + +The `Firefox Profiler `__ is a sophisticated +web-based tool originally built for profiling Firefox itself. It provides +features beyond basic flame graphs, including a timeline view, call tree +exploration, and marker visualization. See the +`Firefox Profiler documentation `__ for +detailed usage instructions. + +To use the output, open the Firefox Profiler in your browser and load the +JSON file. The profiler runs entirely client-side, so your profiling data +never leaves your machine. + +Gecko format automatically collects additional metadata about GIL state and +CPU activity, enabling analysis features specific to Python's threading model. +The profiler emits interval markers that appear as colored bands in the +Firefox Profiler timeline: + +- **GIL markers**: show when threads hold or release the global interpreter lock +- **CPU markers**: show when threads are executing on CPU versus idle +- **Code type markers**: distinguish Python code from native (C extension) code +- **GC markers**: indicate garbage collection activity + +For this reason, the :option:`--mode` option is not available with Gecko format; +all relevant data is captured automatically. + +.. figure:: tachyon-gecko-calltree.png + :alt: Firefox Profiler Call Tree view + :align: center + :width: 100% + + The Call Tree view shows the complete call hierarchy with sample counts + and percentages. The sidebar displays detailed statistics for the + selected function including running time and sample distribution. + +.. figure:: tachyon-gecko-flamegraph.png + :alt: Firefox Profiler Flame Graph view + :align: center + :width: 100% + + The Flame Graph visualization shows call stacks as nested rectangles. + Functions names are visible in the call hierarchy. + +.. figure:: tachyon-gecko-opcodes.png + :alt: Firefox Profiler Marker Chart with opcodes + :align: center + :width: 100% + + The Marker Chart displays interval markers including CPU state, GIL + status, and opcodes. With ``--opcodes`` enabled, bytecode instructions + like ``BINARY_OP_ADD_FLOAT``, ``CALL_PY_EXACT_ARGS``, and + ``CALL_LIST_APPEND`` appear as markers showing execution over time. + + +Heatmap format +-------------- + +Heatmap format (:option:`--heatmap`) generates an interactive HTML visualization +showing sample counts at the source line level:: + + python -m profiling.sampling run --heatmap script.py + python -m profiling.sampling run --heatmap -o my_heatmap script.py + +.. figure:: tachyon-heatmap.png + :alt: Tachyon heatmap visualization + :align: center + :width: 100% + + The heatmap overlays sample counts directly on your source code. Lines are + color-coded from cool (few samples) to hot (many samples). Navigation + buttons (▲▼) let you jump between callers and callees. + +Unlike other formats that produce a single file, heatmap output creates a +directory containing HTML files for each profiled source file. If no output +path is specified, the directory is named ``heatmap_PID``. + +The heatmap visualization displays your source code with a color gradient +indicating how many samples were collected at each line. Hot lines (many +samples) appear in warm colors, while cold lines (few or no samples) appear +in cool colors. This view helps pinpoint exactly which lines of code are +responsible for time consumption. + +The heatmap interface provides several interactive features: + +- **Coloring modes**: toggle between "Self Time" (direct execution) and + "Total Time" (cumulative, including time in called functions) +- **Cold code filtering**: show all lines or only lines with samples +- **Call graph navigation**: each line shows navigation buttons (▲ for callers, + ▼ for callees) that let you trace execution paths through your code. When + multiple functions called or were called from a line, a menu appears showing + all options with their sample counts. +- **Scroll minimap**: a vertical overview showing the heat distribution across + the entire file +- **Hierarchical index**: files organized by type (stdlib, site-packages, + project) with aggregate sample counts per folder +- **Dark/light theme**: toggle with preference saved across sessions +- **Line linking**: click line numbers to create shareable URLs + +When opcode-level profiling is enabled with :option:`--opcodes`, each hot line +can be expanded to show which bytecode instructions consumed time: + +.. figure:: tachyon-heatmap-with-opcodes.png + :alt: Heatmap with expanded bytecode panel + :align: center + :width: 100% + + Expanding a hot line reveals the bytecode instructions executed, including + specialized variants. The panel shows sample counts per instruction and the + overall specialization percentage for the line. + +.. only:: html + + `Try the interactive example <../_static/tachyon-example-heatmap.html>`__! + +Heatmaps are especially useful when you know which file contains a performance +issue but need to identify the specific lines. Many developers prefer this +format because it maps directly to their source code, making it easy to read +and navigate. For smaller scripts and focused analysis, heatmaps provide an +intuitive view that shows exactly where time is spent without requiring +interpretation of hierarchical visualizations. + + +Binary format +------------- + +Binary format (:option:`--binary`) produces a compact binary file for efficient +storage of profiling data:: + + python -m profiling.sampling run --binary -o profile.bin script.py + python -m profiling.sampling attach --binary -o profile.bin 12345 + +The :option:`--compression` option controls data compression: + +- ``auto`` (default): Use zstd compression if available, otherwise no + compression +- ``zstd``: Force zstd compression (requires :mod:`compression.zstd` support) +- ``none``: Disable compression + +:: + + python -m profiling.sampling run --binary --compression=zstd -o profile.bin script.py + +To analyze binary profiles, use the :ref:`replay-command` to convert them to +other formats like flame graphs or pstats output. + + +Record and replay workflow +========================== + +The binary format combined with the replay command enables a record-and-replay +workflow that separates data capture from analysis. Rather than generating +visualizations during profiling, you capture raw data to a compact binary file +and convert it to different formats later. + +This approach has three main benefits: + +- Sampling runs faster because the work of building data structures for + visualization is deferred until replay. +- A single binary capture can be converted to multiple output formats + without re-profiling: pstats for a quick overview, flame graph for visual + exploration, heatmap for line-level detail. +- Binary files are compact and easy to share with colleagues who can convert + them to their preferred format. + +A typical workflow:: + + # Capture profile in production or during tests + python -m profiling.sampling attach --binary -o profile.bin 12345 + + # Later, analyze with different formats + python -m profiling.sampling replay profile.bin + python -m profiling.sampling replay --flamegraph -o profile.html profile.bin + python -m profiling.sampling replay --heatmap -o heatmap profile.bin + + +Live mode +========= + +Live mode (:option:`--live`) provides a terminal-based real-time view of profiling +data, similar to the ``top`` command for system processes:: + + python -m profiling.sampling run --live script.py + python -m profiling.sampling attach --live 12345 + +.. only:: not latex + + .. figure:: tachyon-live-mode-2.gif + :alt: Tachyon live mode showing all threads + :align: center + :width: 100% + + Live mode displays real-time profiling statistics, showing combined + data from multiple threads in a multi-threaded application. + +The display updates continuously as new samples arrive, showing the current +hottest functions. This mode requires the :mod:`curses` module, which is +available on Unix-like systems but not on Windows. The terminal must be at +least 60 columns wide and 12 lines tall; larger terminals display more columns. + +The header displays the top 3 hottest functions, sampling efficiency metrics, +and thread status statistics (GIL held percentage, CPU usage, GC time). The +main table shows function statistics with the currently sorted column indicated +by an arrow (▼). + +When :option:`--opcodes` is enabled, an additional opcode panel appears below the +main table, showing instruction-level statistics for the currently selected +function. This panel displays which bytecode instructions are executing most +frequently, including specialized variants and their base opcodes. + +.. only:: not latex + + .. figure:: tachyon-live-mode-1.gif + :alt: Tachyon live mode with opcode panel + :align: center + :width: 100% + + Live mode with ``--opcodes`` enabled shows an opcode panel with a bytecode + instruction breakdown for the selected function. + + +Keyboard commands +----------------- + +Within live mode, keyboard commands control the display: + +:kbd:`q` + Quit the profiler and return to the shell. + +:kbd:`s` / :kbd:`S` + Cycle through sort orders forward/backward (sample count, percentage, + total time, cumulative percentage, cumulative time). + +:kbd:`p` + Pause or resume display updates. Sampling continues in the background + while the display is paused, so you can freeze the view to examine results + without stopping data collection. + +:kbd:`r` + Reset all statistics and start fresh. This is disabled after profiling + finishes to prevent accidental data loss. + +:kbd:`/` + Enter filter mode to search for functions by name. The filter uses + case-insensitive substring matching against the filename and function name. + Type a pattern and press Enter to apply, or Escape to cancel. Glob patterns + and regular expressions are not supported. + +:kbd:`c` + Clear the current filter and show all functions again. + +:kbd:`t` + Toggle between viewing all threads combined or per-thread statistics. + In per-thread mode, a thread counter (for example, ``1/4``) appears showing + your position among the available threads. + +:kbd:`←` :kbd:`→` or :kbd:`↑` :kbd:`↓` + In per-thread view, navigate between threads. Navigation wraps around + from the last thread to the first and vice versa. + +:kbd:`+` / :kbd:`-` + Increase or decrease the display refresh rate. The range is 0.05 seconds + (20 Hz, very responsive) to 1.0 second (1 Hz, lower overhead). Faster refresh + rates use more CPU. The default is 0.1 seconds (10 Hz). + +:kbd:`x` + Toggle trend indicators that show whether functions are becoming hotter + or cooler over time. When enabled, increasing metrics appear in green and + decreasing metrics appear in red, comparing each update to the previous one. + +:kbd:`h` or :kbd:`?` + Show the help screen with all available commands. + +:kbd:`j` / :kbd:`k` (or :kbd:`Up` / :kbd:`Down`) + Navigate through opcode entries in the opcode panel (when ``--opcodes`` is + enabled). These keys scroll through the instruction-level statistics for the + currently selected function. + +When profiling finishes (duration expires or target process exits), the display +shows a "PROFILING COMPLETE" banner and freezes the final results. You can +still navigate, sort, and filter the results before pressing :kbd:`q` to exit. + +Live mode is incompatible with output format options (:option:`--collapsed`, +:option:`--flamegraph`, and so on) because it uses an interactive terminal +interface rather than producing file output. + + +Async-aware profiling +===================== + +For programs using :mod:`asyncio`, the profiler offers async-aware mode +(:option:`--async-aware`) that reconstructs call stacks based on the task structure +rather than the raw Python frames:: + + python -m profiling.sampling run --async-aware async_script.py + +Standard profiling of async code can be confusing because the physical call +stack often shows event loop internals rather than the logical flow of your +coroutines. Async-aware mode addresses this by tracking which task is running +and presenting stacks that reflect the ``await`` chain. + +.. code-block:: python + + import asyncio + + async def fetch(url): + await asyncio.sleep(0.1) + return url + + async def main(): + for _ in range(50): + await asyncio.gather(fetch("a"), fetch("b"), fetch("c")) + + if __name__ == "__main__": + asyncio.run(main()) + +:: + + python -m profiling.sampling run --async-aware --flamegraph -o out.html script.py + +.. note:: + + Async-aware profiling requires the target process to have the :mod:`asyncio` + module loaded. If you profile a script before it imports asyncio, async-aware + mode will not be able to capture task information. + + +Async modes +----------- + +The :option:`--async-mode` option controls which tasks appear in the profile:: + + python -m profiling.sampling run --async-aware --async-mode=running async_script.py + python -m profiling.sampling run --async-aware --async-mode=all async_script.py + +With :option:`--async-mode`\ ``=running`` (the default), only the task currently executing +on the CPU is profiled. This shows where your program is actively spending time +and is the typical choice for performance analysis. + +With :option:`--async-mode`\ ``=all``, tasks that are suspended (awaiting I/O, locks, or +other tasks) are also included. This mode is useful for understanding what your +program is waiting on, but produces larger profiles since every suspended task +appears in each sample. + + +Task markers and stack reconstruction +------------------------------------- + +In async-aware profiles, you will see ```` frames that mark boundaries +between asyncio tasks. These are synthetic frames inserted by the profiler to +show the task structure. The task name appears as the function name in these +frames. + +When a task awaits another task, the profiler reconstructs the logical call +chain by following the ``await`` relationships. Only "leaf" tasks (tasks that +no other task is currently awaiting) generate their own stack entries. Tasks +being awaited by other tasks appear as part of their awaiter's stack instead. + +If a task has multiple awaiters (a diamond pattern in the task graph), the +profiler deterministically selects one parent and annotates the task marker +with the number of parents, for example ``MyTask (2 parents)``. This indicates +that alternate execution paths exist but are not shown in this particular stack. + + +Option restrictions +------------------- + +Async-aware mode uses a different stack reconstruction mechanism and is +incompatible with: :option:`--native`, :option:`--no-gc`, :option:`--all-threads`, and +:option:`--mode`\ ``=cpu`` or :option:`--mode`\ ``=gil``. + + +Command-line interface +====================== + +.. program:: profiling.sampling + +The complete command-line interface for reference. + + +Global options +-------------- + +.. option:: run + + Run and profile a Python script or module. + +.. option:: attach + + Attach to and profile a running process by PID. + +.. option:: dump + + Print a single one-shot snapshot of a running process's Python stack. + +.. option:: replay + + Convert a binary profile file to another output format. + + +Dump options +------------ + +The following options apply to the ``dump`` subcommand: + +.. option:: -a, --all-threads + + Dump all threads in the target process instead of just the main thread. + +.. option:: --native + + Include ```` frames for non-Python code. + +.. option:: --no-gc + + Exclude ```` frames for active garbage collection. + +.. option:: --opcodes + + Show bytecode opcode names when available. + +.. option:: --async-aware + + Reconstruct the stack across ``await`` boundaries for asyncio + applications. + +.. option:: --async-mode + + Async stack mode: ``running`` (only the running task) or ``all`` + (all tasks including waiting). Defaults to ``all`` for ``dump``. + Requires :option:`--async-aware`. + +.. option:: --blocking + + Pause all threads in the target process while reading the stack. + + +Sampling options +---------------- + +.. option:: -r , --sampling-rate + + Sampling rate (for example, ``10000``, ``10khz``, ``10k``). Default: ``1khz``. + +.. option:: -d , --duration + + Profiling duration in seconds. Default: run to completion. + +.. option:: -a, --all-threads + + Sample all threads, not just the main thread. + +.. option:: --realtime-stats + + Display sampling statistics during profiling. + +.. option:: --native + + Include ```` frames for non-Python code. + +.. option:: --no-gc + + Exclude ```` frames for garbage collection. + +.. option:: --async-aware + + Enable async-aware profiling for asyncio programs. + +.. option:: --opcodes + + Gather bytecode opcode information for instruction-level profiling. Shows + which bytecode instructions are executing, including specializations. + Compatible with ``--live``, ``--flamegraph``, ``--heatmap``, and ``--gecko`` + formats only. + +.. option:: --subprocesses + + Also profile subprocesses. Each subprocess gets its own profiler + instance and output file. Incompatible with ``--live``. + +.. option:: --blocking + + Pause the target process during each sample. This ensures consistent + stack traces at the cost of slowing down the target. Use with longer + intervals (1000 µs or higher) to minimize impact. See :ref:`blocking-mode` + for details. + + +Mode options +------------ + +.. option:: --mode + + Sampling mode: ``wall`` (default), ``cpu``, ``gil``, or ``exception``. + The ``cpu``, ``gil``, and ``exception`` modes are incompatible with + ``--async-aware``. + +.. option:: --async-mode + + Async profiling mode: ``running`` (default) or ``all``. + Requires ``--async-aware``. + + +Output options +-------------- + +.. option:: --pstats + + Generate pstats statistics. This is the default. + When written to stdout, the output is a text table; with :option:`-o`, + it is a binary pstats file. + +.. option:: --collapsed + + Generate collapsed stack format for external flame graph tools. + +.. option:: --flamegraph + + Generate self-contained HTML flame graph. + +.. option:: --diff-flamegraph + + Generate differential flamegraph comparing to a baseline binary profile. + +.. option:: --gecko + + Generate Gecko JSON format for Firefox Profiler. + +.. option:: --heatmap + + Generate HTML heatmap with line-level sample counts. + +.. option:: --binary + + Generate high-performance binary format for later conversion with the + ``replay`` command. + +.. option:: --compression + + Compression for binary format: ``auto`` (use zstd if available, default), + ``zstd``, or ``none``. + +.. option:: -o , --output + + Output file or directory path. Default behavior varies by format: + :option:`--pstats` prints a text table to stdout, while ``-o`` writes a + binary pstats file. Other formats generate a file named + ``_.`` (for example, ``flamegraph_12345.html``). + :option:`--heatmap` creates a directory named ``heatmap_``. + +.. option:: --browser + + Automatically open HTML output (:option:`--flamegraph` and + :option:`--heatmap`) in your default web browser after generation. + When profiling with :option:`--subprocesses`, only the main process + opens the browser; subprocess outputs are never auto-opened. + + +pstats display options +---------------------- + +These options apply only to pstats format output. + +.. option:: --sort + + Sort order: ``nsamples``, ``tottime``, ``cumtime``, ``sample-pct``, + ``cumul-pct``, ``nsamples-cumul``, or ``name``. Default: ``nsamples``. + +.. option:: -l , --limit + + Maximum number of entries to display. Default: 15. + +.. option:: --no-summary + + Omit the Legend and Summary of Interesting Functions sections from output. + + +Run command options +------------------- + +.. option:: -m, --module + + Treat the target as a module name rather than a script path. + +.. option:: --live + + Start interactive terminal interface instead of batch profiling. + + +.. seealso:: + + :mod:`profiling` + Overview of Python profiling tools and guidance on choosing a profiler. + + :mod:`profiling.tracing` + Deterministic tracing profiler for exact call counts and timing. + + :mod:`pstats` + Statistics analysis for profile data. + + `Firefox Profiler `__ + Web-based profiler that accepts Gecko format output. See the + `documentation `__ for usage details. + + `FlameGraph `__ + Tools for generating flame graphs from collapsed stack format. diff --git a/Doc/library/profiling.tracing.rst b/Doc/library/profiling.tracing.rst new file mode 100644 index 000000000000000..d45423cf0d8a72d --- /dev/null +++ b/Doc/library/profiling.tracing.rst @@ -0,0 +1,331 @@ +.. _profiling-tracing: + +**************************************************** +:mod:`!profiling.tracing` --- Deterministic profiler +**************************************************** + +.. module:: profiling.tracing + :synopsis: Deterministic tracing profiler for Python programs. + +.. module:: cProfile + :synopsis: Alias for profiling.tracing (backward compatibility). + :noindex: + +.. versionadded:: 3.15 + +**Source code:** :source:`Lib/profiling/tracing/` + +-------------- + +The :mod:`!profiling.tracing` module provides deterministic profiling of Python +programs. It monitors every function call, function return, and exception event, +recording precise timing for each. This approach provides exact call counts and +complete visibility into program execution, making it ideal for development and +testing scenarios. + +.. note:: + + This module is also available as ``cProfile`` for backward compatibility. + The ``cProfile`` name will continue to work in all future Python versions. + Use whichever import style suits your codebase:: + + # Preferred (new style) + import profiling.tracing + profiling.tracing.run('my_function()') + + # Also works (backward compatible) + import cProfile + cProfile.run('my_function()') + + +What is deterministic profiling? +================================ + +:dfn:`Deterministic profiling` captures every function call, function return, +and exception event during program execution. The profiler measures the precise +time intervals between these events, providing exact statistics about how the +program behaves. + +In contrast to :ref:`statistical profiling `, which samples +the call stack periodically to estimate where time is spent, deterministic +profiling records every event. This means you get exact call counts rather than +statistical approximations. The trade-off is that instrumenting every event +introduces overhead that can slow down program execution. + +Python's interpreted nature makes deterministic profiling practical. The +interpreter already dispatches events for function calls and returns, so the +profiler can hook into this mechanism without requiring code modification. The +overhead tends to be moderate relative to the inherent cost of interpretation, +making deterministic profiling suitable for most development workflows. + +Deterministic profiling helps answer questions like: + +- How many times was this function called? +- What is the complete call graph of my program? +- Which functions are called by a particular function? +- Are there unexpected function calls happening? + +Call count statistics can identify bugs (surprising counts) and inline +expansion opportunities (high call counts). Internal time statistics reveal +"hot loops" that warrant optimization. Cumulative time statistics help identify +algorithmic inefficiencies. The handling of cumulative times in this profiler +allows direct comparison of recursive and iterative implementations. + + +.. _profiling-tracing-cli: + +Command-line interface +====================== + +.. program:: profiling.tracing + +The :mod:`!profiling.tracing` module can be invoked as a script to profile +another script or module: + +.. code-block:: shell-session + + python -m profiling.tracing [-o output_file] [-s sort_order] (-m module | script.py) + +This runs the specified script or module under the profiler and prints the +results to standard output (or saves them to a file). + +.. option:: -o + + Write the profile results to a file instead of standard output. The output + file can be read by the :mod:`pstats` module for later analysis. + +.. option:: -s + + Sort the output by the specified key. This accepts any of the sort keys + recognized by :meth:`pstats.Stats.sort_stats`, such as ``cumulative``, + ``time``, ``calls``, or ``name``. This option only applies when + :option:`-o ` is not specified. + +.. option:: -m + + Profile a module instead of a script. The module is located using the + standard import mechanism. + + .. versionadded:: 3.7 + The ``-m`` option for ``cProfile``. + + .. versionadded:: 3.8 + The ``-m`` option for :mod:`profile`. + + +Programmatic usage examples +=========================== + +For more control over profiling, use the module's functions and classes +directly. + + +Basic profiling +--------------- + +The simplest approach uses the :func:`!run` function:: + + import profiling.tracing + profiling.tracing.run('my_function()') + +This profiles the given code string and prints a summary to standard output. +To save results for later analysis:: + + profiling.tracing.run('my_function()', 'output.prof') + + +Using the :class:`!Profile` class +--------------------------------- + +The :class:`!Profile` class provides fine-grained control:: + + import profiling.tracing + import pstats + from io import StringIO + + pr = profiling.tracing.Profile() + pr.enable() + # ... code to profile ... + pr.disable() + + # Print results + s = StringIO() + ps = pstats.Stats(pr, stream=s).sort_stats(pstats.SortKey.CUMULATIVE) + ps.print_stats() + print(s.getvalue()) + +The :class:`!Profile` class also works as a context manager:: + + import profiling.tracing + + with profiling.tracing.Profile() as pr: + # ... code to profile ... + + pr.print_stats() + + +Module reference +================ + +.. currentmodule:: profiling.tracing + +.. function:: run(command, filename=None, sort=-1) + + Profile execution of a command and print or save the results. + + This function executes the *command* string using :func:`exec` in the + ``__main__`` module's namespace:: + + exec(command, __main__.__dict__, __main__.__dict__) + + If *filename* is not provided, the function creates a :class:`pstats.Stats` + instance and prints a summary to standard output. If *filename* is + provided, the raw profile data is saved to that file for later analysis + with :mod:`pstats`. + + The *sort* argument specifies the sort order for printed output, accepting + any value recognized by :meth:`pstats.Stats.sort_stats`. + + +.. function:: runctx(command, globals, locals, filename=None, sort=-1) + + Profile execution of a command with explicit namespaces. + + Like :func:`run`, but executes the command with the specified *globals* + and *locals* mappings instead of using the ``__main__`` module's namespace:: + + exec(command, globals, locals) + + +.. class:: Profile(timer=None, timeunit=0.0, subcalls=True, builtins=True) + + A profiler object that collects execution statistics. + + The optional *timer* argument specifies a custom timing function. If not + provided, the profiler uses a platform-appropriate default timer. When + supplying a custom timer, it must return a single number representing the + current time. If the timer returns integers, use *timeunit* to specify the + duration of one time unit (for example, ``0.001`` for milliseconds). + + The *subcalls* argument controls whether the profiler tracks call + relationships between functions. The *builtins* argument controls whether + built-in functions are profiled. + + .. versionchanged:: 3.8 + Added context manager support. + + .. method:: enable() + + Start collecting profiling data. + + .. method:: disable() + + Stop collecting profiling data. + + .. method:: create_stats() + + Stop collecting data and record the results internally as the current + profile. + + .. method:: print_stats(sort=-1) + + Create a :class:`pstats.Stats` object from the current profile and print + the results to standard output. + + The *sort* argument specifies the sorting order. It accepts a single + key or a tuple of keys for multi-level sorting, using the same values + as :meth:`pstats.Stats.sort_stats`. + + .. versionadded:: 3.13 + Support for a tuple of sort keys. + + .. method:: dump_stats(filename) + + Write the current profile data to *filename*. The file can be read by + :class:`pstats.Stats` for later analysis. + + .. method:: run(cmd) + + Profile the command string via :func:`exec`. + + .. method:: runctx(cmd, globals, locals) + + Profile the command string via :func:`exec` with the specified + namespaces. + + .. method:: runcall(func, /, *args, **kwargs) + + Profile a function call. Returns whatever *func* returns:: + + result = pr.runcall(my_function, arg1, arg2, keyword=value) + +.. note:: + + Profiling requires that the profiled code returns normally. If the + interpreter terminates (for example, via :func:`sys.exit`) during + profiling, no results will be available. + + +Using a custom timer +==================== + +The :class:`Profile` class accepts a custom timing function, allowing you to +measure different aspects of execution such as wall-clock time or CPU time. +Pass the timing function to the constructor:: + + pr = profiling.tracing.Profile(my_timer_function) + +The timer function must return a single number representing the current time. +If it returns integers, also specify *timeunit* to indicate the duration of +one unit:: + + # Timer returns time in milliseconds + pr = profiling.tracing.Profile(my_ms_timer, 0.001) + +For best performance, the timer function should be as fast as possible. The +profiler calls it frequently, so timer overhead directly affects profiling +overhead. + +The :mod:`time` module provides several functions suitable for use as custom +timers: + +- :func:`time.perf_counter` for high-resolution wall-clock time +- :func:`time.process_time` for CPU time (excluding sleep) +- :func:`time.monotonic` for monotonic clock time + + +Limitations +=========== + +Deterministic profiling has inherent limitations related to timing accuracy. + +The underlying timer typically has a resolution of about one millisecond. +Measurements cannot be more accurate than this resolution. With enough +measurements, timing errors tend to average out, but individual measurements +may be imprecise. + +There is also latency between when an event occurs and when the profiler +captures the timestamp. Similarly, there is latency after reading the +timestamp before user code resumes. Functions called frequently accumulate +this latency, which can make them appear slower than they actually are. This +error is typically less than one clock tick per call but can become +significant for functions called many times. + +The :mod:`!profiling.tracing` module (and its ``cProfile`` alias) is +implemented as a C extension with low overhead, so these timing issues are +less pronounced than with the deprecated pure Python :mod:`profile` module. + + +.. seealso:: + + :mod:`profiling` + Overview of Python profiling tools and guidance on choosing a profiler. + + :mod:`profiling.sampling` + Statistical sampling profiler for production use. + + :mod:`pstats` + Statistics analysis and formatting for profile data. + + :mod:`profile` + Deprecated pure Python profiler (includes calibration documentation). diff --git a/Doc/library/pstats.rst b/Doc/library/pstats.rst new file mode 100644 index 000000000000000..585f17bdb99a709 --- /dev/null +++ b/Doc/library/pstats.rst @@ -0,0 +1,362 @@ +.. _pstats-module: + +******************************************* +:mod:`!pstats` --- Statistics for profilers +******************************************* + +.. module:: pstats + :synopsis: Statistics object for analyzing profiler output. + +**Source code:** :source:`Lib/pstats.py` + +-------------- + +The :mod:`!pstats` module provides tools for reading, manipulating, and +displaying profiling statistics generated by Python's profilers. It reads +output from both :mod:`profiling.tracing` (deterministic profiler) and +:mod:`profiling.sampling` (statistical profiler). + + +Reading and displaying profile data +=================================== + +The :class:`Stats` class is the primary interface for working with profile +data. It can read statistics from files or directly from a +:class:`~profiling.tracing.Profile` object. + +Load statistics from a file and print a basic report:: + + import pstats + + p = pstats.Stats('profile_output.prof') + p.print_stats() + +The :class:`Stats` object provides methods for sorting and filtering the +data before printing. For example, to see the ten functions with the highest +cumulative time:: + + from pstats import SortKey + + p = pstats.Stats('profile_output.prof') + p.sort_stats(SortKey.CUMULATIVE).print_stats(10) + + +Working with statistics +----------------------- + +The :class:`Stats` class supports method chaining, making it convenient to +perform multiple operations:: + + p = pstats.Stats('restats') + p.strip_dirs().sort_stats(-1).print_stats() + +The :meth:`~Stats.strip_dirs` method removes directory paths from filenames, +making the output more compact. The :meth:`~Stats.sort_stats` method accepts +various keys to control the sort order. + +Different sort keys highlight different aspects of performance:: + + from pstats import SortKey + + # Functions that consume the most cumulative time + p.sort_stats(SortKey.CUMULATIVE).print_stats(10) + + # Functions that consume the most time in their own code + p.sort_stats(SortKey.TIME).print_stats(10) + + # Functions sorted by name + p.sort_stats(SortKey.NAME).print_stats() + + +Filtering output +---------------- + +The :meth:`~Stats.print_stats` method accepts restrictions that filter +which functions are displayed. Restrictions can be integers (limiting the +count), floats between 0 and 1 (selecting a percentage), or strings (matching +function names via regular expression). + +Print only the top 10%:: + + p.print_stats(.1) + +Print only functions whose names contain "init":: + + p.print_stats('init') + +Combine restrictions (they apply sequentially):: + + # Top 10%, then only those containing "init" + p.print_stats(.1, 'init') + + # Functions in files matching "foo:", limited to top 50% + p.sort_stats(SortKey.FILENAME).print_stats('foo:', .5) + + +Analyzing call relationships +---------------------------- + +The :meth:`~Stats.print_callers` method shows which functions called each +displayed function:: + + p.print_callers() + +The :meth:`~Stats.print_callees` method shows the opposite relationship, +listing which functions each displayed function called:: + + p.print_callees() + +Both methods accept the same restriction arguments as :meth:`~Stats.print_stats`. + + +Combining multiple profiles +--------------------------- + +Statistics from multiple profiling runs can be combined into a single +:class:`Stats` object:: + + # Load multiple files at once + p = pstats.Stats('run1.prof', 'run2.prof', 'run3.prof') + + # Or add files incrementally + p = pstats.Stats('run1.prof') + p.add('run2.prof') + p.add('run3.prof') + +When files are combined, statistics for identical functions (same file, line, +and name) are accumulated, giving an aggregate view across all profiling runs. + + +The :class:`!Stats` class +========================= + +.. class:: Stats(*filenames_or_profile, stream=sys.stdout) + + Create a statistics object from profile data. + + The arguments can be filenames (strings or path-like objects) or + :class:`~profiling.tracing.Profile` objects. If multiple sources are + provided, their statistics are combined. + + The *stream* argument specifies where output from :meth:`print_stats` and + related methods is written. It defaults to :data:`sys.stdout`. + + The profile data format is specific to the Python version that created it. + There is no compatibility guarantee between Python versions or between + different profilers. + + .. method:: strip_dirs() + + Remove leading path information from all filenames. + + This method modifies the object in place and returns it for method + chaining. After stripping, the statistics are considered to be in + random order. + + If stripping causes two functions to become indistinguishable (same + filename, line number, and function name), their statistics are + combined into a single entry. + + .. method:: add(*filenames) + + Add profiling data from additional files. + + The files must have been created by the same profiler type. Statistics + for identical functions are accumulated. + + .. method:: dump_stats(filename) + + Save the current statistics to a file. + + The file is created if it does not exist and overwritten if it does. + The saved data can be loaded by creating a new :class:`Stats` object. + + .. method:: sort_stats(*keys) + + Sort the statistics according to the specified criteria. + + Each key can be a string or a :class:`SortKey` enum member. When + multiple keys are provided, later keys break ties in earlier keys. + + Using :class:`SortKey` enum members is preferred over strings as it + provides better error checking:: + + from pstats import SortKey + p.sort_stats(SortKey.CUMULATIVE) + + Valid sort keys: + + +------------------+------------------------+----------------------+ + | String | Enum | Meaning | + +==================+========================+======================+ + | ``'calls'`` | ``SortKey.CALLS`` | call count | + +------------------+------------------------+----------------------+ + | ``'cumulative'`` | ``SortKey.CUMULATIVE`` | cumulative time | + +------------------+------------------------+----------------------+ + | ``'cumtime'`` | N/A | cumulative time | + +------------------+------------------------+----------------------+ + | ``'file'`` | N/A | file name | + +------------------+------------------------+----------------------+ + | ``'filename'`` | ``SortKey.FILENAME`` | file name | + +------------------+------------------------+----------------------+ + | ``'module'`` | N/A | file name | + +------------------+------------------------+----------------------+ + | ``'ncalls'`` | N/A | call count | + +------------------+------------------------+----------------------+ + | ``'pcalls'`` | ``SortKey.PCALLS`` | primitive call count | + +------------------+------------------------+----------------------+ + | ``'line'`` | ``SortKey.LINE`` | line number | + +------------------+------------------------+----------------------+ + | ``'name'`` | ``SortKey.NAME`` | function name | + +------------------+------------------------+----------------------+ + | ``'nfl'`` | ``SortKey.NFL`` | name/file/line | + +------------------+------------------------+----------------------+ + | ``'stdname'`` | ``SortKey.STDNAME`` | standard name | + +------------------+------------------------+----------------------+ + | ``'time'`` | ``SortKey.TIME`` | internal time | + +------------------+------------------------+----------------------+ + | ``'tottime'`` | N/A | internal time | + +------------------+------------------------+----------------------+ + + All sorts on statistics are in descending order (most time consuming + first), while name, file, and line number sorts are ascending + (alphabetical). + + The difference between ``SortKey.NFL`` and ``SortKey.STDNAME`` is that + NFL sorts line numbers numerically while STDNAME sorts them as strings. + ``sort_stats(SortKey.NFL)`` is equivalent to + ``sort_stats(SortKey.NAME, SortKey.FILENAME, SortKey.LINE)``. + + For backward compatibility, the numeric arguments ``-1``, ``0``, ``1``, + and ``2`` are also accepted, meaning ``'stdname'``, ``'calls'``, + ``'time'``, and ``'cumulative'`` respectively. + + .. versionadded:: 3.7 + The :class:`SortKey` enum. + + .. method:: reverse_order() + + Reverse the current sort order. + + By default, the sort direction is chosen appropriately for the sort key + (descending for time-based keys, ascending for name-based keys). This + method inverts that choice. + + .. method:: print_stats(*restrictions) + + Print a report of the profiling statistics. + + The output includes a header line summarizing the data, followed by a + table of function statistics sorted according to the last + :meth:`sort_stats` call. + + Restrictions filter the output. Each restriction is either: + + - An integer: limits output to that many entries + - A float between 0.0 and 1.0: selects that fraction of entries + - A string: matches function names via regular expression + + Restrictions are applied sequentially. For example:: + + print_stats(.1, 'foo:') + + First limits to the top 10%, then filters to functions matching 'foo:'. + + .. method:: print_callers(*restrictions) + + Print the callers of each function in the statistics. + + For each function in the filtered results, shows which functions called + it and how often. + + With :mod:`profiling.tracing` (or ``cProfile``), each caller line + shows three numbers: the number of calls from that caller, and the + total and cumulative times for those specific calls. + + Accepts the same restriction arguments as :meth:`print_stats`. + + .. method:: print_callees(*restrictions) + + Print the functions called by each function in the statistics. + + This is the inverse of :meth:`print_callers`, showing which functions + each listed function called. + + Accepts the same restriction arguments as :meth:`print_stats`. + + .. method:: get_stats_profile() + + Return a ``StatsProfile`` object containing the statistics. + + The returned object provides programmatic access to the profile data, + with function names mapped to ``FunctionProfile`` objects + containing timing and call count information. + + .. versionadded:: 3.9 + + +.. class:: SortKey + + An enumeration of valid sort keys for :meth:`Stats.sort_stats`. + + .. attribute:: CALLS + + Sort by call count. + + .. attribute:: CUMULATIVE + + Sort by cumulative time. + + .. attribute:: FILENAME + + Sort by file name. + + .. attribute:: LINE + + Sort by line number. + + .. attribute:: NAME + + Sort by function name. + + .. attribute:: NFL + + Sort by name, then file, then line number (numeric line sort). + + .. attribute:: PCALLS + + Sort by primitive (non-recursive) call count. + + .. attribute:: STDNAME + + Sort by standard name (string-based line sort). + + .. attribute:: TIME + + Sort by internal time (time in function excluding subcalls). + + +.. _pstats-cli: + +Command-line interface +====================== + +The :mod:`!pstats` module can be invoked as a script to interactively browse +profile data:: + + python -m pstats profile_output.prof + +This opens a line-oriented interface (built on :mod:`cmd`) for examining the +statistics. Type ``help`` at the prompt for available commands. + + +.. seealso:: + + :mod:`profiling` + Overview of Python profiling tools. + + :mod:`profiling.tracing` + Deterministic tracing profiler. + + :mod:`profiling.sampling` + Statistical sampling profiler. diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index 1a44bb13a841de6..a7be5779fb26208 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -2,17 +2,13 @@ ========================================= .. module:: pty - :platform: Unix :synopsis: Pseudo-Terminal Handling for Unix. -.. moduleauthor:: Steen Lumholt -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/pty.py` -------------- -The :mod:`pty` module defines operations for handling the pseudo-terminal +The :mod:`!pty` module defines operations for handling the pseudo-terminal concept: starting another process and being able to write to and read from its controlling terminal programmatically. @@ -22,7 +18,7 @@ Pseudo-terminal handling is highly platform dependent. This code is mainly tested on Linux, FreeBSD, and macOS (it is supposed to work on other POSIX platforms but it's not been thoroughly tested). -The :mod:`pty` module defines the following functions: +The :mod:`!pty` module defines the following functions: .. function:: fork() @@ -33,9 +29,14 @@ The :mod:`pty` module defines the following functions: file descriptor connected to the child's controlling terminal (and also to the child's standard input and output). + The returned file descriptor *fd* is :ref:`non-inheritable `. + .. warning:: On macOS the use of this function is unsafe when mixed with using higher-level system APIs, and that includes using :mod:`urllib.request`. + .. versionchanged:: 3.15 + The returned file descriptor is now made non-inheritable. + .. function:: openpty() @@ -88,8 +89,6 @@ The :mod:`pty` module defines the following functions: Example ------- -.. sectionauthor:: Steen Lumholt - The following program acts like the Unix command :manpage:`script(1)`, using a pseudo-terminal to record all input and output of a terminal session in a "typescript". :: diff --git a/Doc/library/pwd.rst b/Doc/library/pwd.rst index e1ff32912132f7d..7691fed2c7cb835 100644 --- a/Doc/library/pwd.rst +++ b/Doc/library/pwd.rst @@ -2,7 +2,6 @@ ===================================== .. module:: pwd - :platform: Unix :synopsis: The password database (getpwnam() and friends). -------------- diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index 75aa739d1003b8b..7aa960de3f23454 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -4,16 +4,13 @@ .. module:: py_compile :synopsis: Generate byte-code files from Python source files. -.. sectionauthor:: Fred L. Drake, Jr. -.. documentation based on module docstrings - **Source code:** :source:`Lib/py_compile.py` .. index:: pair: file; byte-code -------------- -The :mod:`py_compile` module provides a function to generate a byte-code file +The :mod:`!py_compile` module provides a function to generate a byte-code file from a source file, and another function used when the module source file is invoked as a script. diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index 5efb11d89dd143d..ed9fc6d0b5cf8c0 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -4,13 +4,11 @@ .. module:: pyclbr :synopsis: Supports information extraction for a Python module browser. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/pyclbr.py` -------------- -The :mod:`pyclbr` module provides limited information about the +The :mod:`!pyclbr` module provides limited information about the functions, classes, and methods defined in a Python-coded module. The information is sufficient to implement a module browser. The information is extracted from the Python source code rather than by diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index e8f153ee1b35ce0..a0cfb440a36ffa9 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -4,9 +4,6 @@ .. module:: pydoc :synopsis: Documentation generator and online help system. -.. moduleauthor:: Ka-Ping Yee -.. sectionauthor:: Ka-Ping Yee - **Source code:** :source:`Lib/pydoc.py` .. index:: @@ -71,6 +68,11 @@ will start a HTTP server on port 1234, allowing you to browse the documentation at ``http://localhost:1234/`` in your preferred web browser. Specifying ``0`` as the port number will select an arbitrary unused port. +.. warning:: + + The :mod:`!pydoc` HTTP server is intended for local use during + development and is not suitable for production use. + :program:`python -m pydoc -n ` will start the server listening at the given hostname. By default the hostname is 'localhost' but if you want the server to be reached from other machines, you may want to change the host name that the diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 5506ac828e5abe7..c88411ce0b7b91f 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -4,8 +4,6 @@ .. module:: xml.parsers.expat :synopsis: An interface to the Expat non-validating XML parser. -.. moduleauthor:: Paul Prescod - -------------- .. Markup notes: @@ -24,7 +22,7 @@ .. index:: single: Expat -The :mod:`xml.parsers.expat` module is a Python interface to the Expat +The :mod:`!xml.parsers.expat` module is a Python interface to the Expat non-validating XML parser. The module provides a single extension type, :class:`xmlparser`, that represents the current state of an XML parser. After an :class:`xmlparser` object has been created, various attributes of the object @@ -55,7 +53,7 @@ This module provides one exception and one type object: The type of the return values from the :func:`ParserCreate` function. -The :mod:`xml.parsers.expat` module contains two functions: +The :mod:`!xml.parsers.expat` module contains two functions: .. function:: ErrorString(errno) @@ -65,12 +63,33 @@ The :mod:`xml.parsers.expat` module contains two functions: .. function:: ParserCreate(encoding=None, namespace_separator=None) - Creates and returns a new :class:`xmlparser` object. *encoding*, if specified, - must be a string naming the encoding used by the XML data. Expat doesn't - support as many encodings as Python does, and its repertoire of encodings can't - be extended; it supports UTF-8, UTF-16, ISO-8859-1 (Latin1), and ASCII. If - *encoding* [1]_ is given it will override the implicit or explicit encoding of the - document. + Creates and returns a new :class:`xmlparser` object. + *encoding* [1]_, if specified, must be a string naming the encoding + used by the XML data. + If it is given it will override the implicit or explicit encoding + of the document. + + .. impl-detail:: + + Expat natively understands and processes UTF-8, UTF-16, UTF-16BE, + UTF-16LE, ISO-8859-1, and US-ASCII. + For other encodings (including aliases like Latin1 and ASCII) it + falls back to Python. + It supports most of 8-bit encodings and many multi-byte encodings + like Shift_JIS, although only BMP characters (``U+0000-U+FFFF``) + are supported with non-native encodings (this restriction is also + applied to aliases like UTF8). + These restrictions only apply if *encoding* is not given. + + .. versionchanged:: next + Added support for multi-byte encodings. + + .. _xmlparser-non-root: + + Parsers created through :func:`!ParserCreate` are called "root" parsers, + in the sense that they do not have any parent parser attached. Non-root + parsers are created by :meth:`parser.ExternalEntityParserCreate + `. Expat can optionally do XML namespace processing for you, enabled by providing a value for *namespace_separator*. The value must be a one-character string; a @@ -108,7 +127,6 @@ The :mod:`xml.parsers.expat` module contains two functions: XML document. Call ``ParserCreate`` for each document to provide unique parser instances. - .. seealso:: `The Expat XML Parser `_ @@ -216,10 +234,10 @@ XMLParser Objects Calling ``SetReparseDeferralEnabled(True)`` allows re-enabling reparse deferral. - Note that :meth:`SetReparseDeferralEnabled` has been backported to some - prior releases of CPython as a security fix. Check for availability of - :meth:`SetReparseDeferralEnabled` using :func:`hasattr` if used in code - running across a variety of Python versions. + :meth:`!SetReparseDeferralEnabled` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. .. versionadded:: 3.13 @@ -231,6 +249,131 @@ XMLParser Objects .. versionadded:: 3.13 +:class:`!xmlparser` objects have the following methods to tune protections +against some common XML vulnerabilities. + +.. method:: xmlparser.SetBillionLaughsAttackProtectionActivationThreshold(threshold, /) + + Sets the number of output bytes needed to activate protection against + `billion laughs`_ attacks. + + The number of output bytes includes amplification from entity expansion + and reading DTD files. + + Parser objects usually have a protection activation threshold of 8 MiB, + but the actual default value depends on the underlying Expat library. + + An :exc:`ExpatError` is raised if this method is called on a + |xml-non-root-parser| parser. + The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset` + should not be used as they may have no special meaning. + + :meth:`!SetBillionLaughsAttackProtectionActivationThreshold` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. + + .. note:: + + Activation thresholds below 4 MiB are known to break support for DITA 1.3 + payload and are hence not recommended. + + .. versionadded:: 3.15 + +.. method:: xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification(max_factor, /) + + Sets the maximum tolerated amplification factor for protection against + `billion laughs`_ attacks. + + The amplification factor is calculated as ``(direct + indirect) / direct`` + while parsing, where ``direct`` is the number of bytes read from + the primary document in parsing and ``indirect`` is the number of + bytes added by expanding entities and reading of external DTD files. + + The *max_factor* value must be a non-NaN :class:`float` value greater than + or equal to 1.0. Peak amplifications of factor 15,000 for the entire payload + and of factor 30,000 in the middle of parsing have been observed with small + benign files in practice. In particular, the activation threshold should be + carefully chosen to avoid false positives. + + Parser objects usually have a maximum amplification factor of 100, + but the actual default value depends on the underlying Expat library. + + An :exc:`ExpatError` is raised if this method is called on a + |xml-non-root-parser| parser or if *max_factor* is outside the valid range. + The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset` + should not be used as they may have no special meaning. + + :meth:`!SetBillionLaughsAttackProtectionMaximumAmplification` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. + + .. note:: + + The maximum amplification factor is only considered if the threshold + that can be adjusted by :meth:`.SetBillionLaughsAttackProtectionActivationThreshold` + is exceeded. + + .. versionadded:: 3.15 + +.. method:: xmlparser.SetAllocTrackerActivationThreshold(threshold, /) + + Sets the number of allocated bytes of dynamic memory needed to activate + protection against disproportionate use of RAM. + + Parser objects usually have an allocation activation threshold of 64 MiB, + but the actual default value depends on the underlying Expat library. + + An :exc:`ExpatError` is raised if this method is called on a + |xml-non-root-parser| parser. + The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset` + should not be used as they may have no special meaning. + + :meth:`!SetAllocTrackerActivationThreshold` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. + + .. versionadded:: 3.15 + +.. method:: xmlparser.SetAllocTrackerMaximumAmplification(max_factor, /) + + Sets the maximum amplification factor between direct input and bytes + of dynamic memory allocated. + + The amplification factor is calculated as ``allocated / direct`` + while parsing, where ``direct`` is the number of bytes read from + the primary document in parsing and ``allocated`` is the number + of bytes of dynamic memory allocated in the parser hierarchy. + + The *max_factor* value must be a non-NaN :class:`float` value greater than + or equal to 1.0. Amplification factors greater than 100.0 can be observed + near the start of parsing even with benign files in practice. In particular, + the activation threshold should be carefully chosen to avoid false positives. + + Parser objects usually have a maximum amplification factor of 100, + but the actual default value depends on the underlying Expat library. + + An :exc:`ExpatError` is raised if this method is called on a + |xml-non-root-parser| parser or if *max_factor* is outside the valid range. + The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset` + should not be used as they may have no special meaning. + + :meth:`!SetAllocTrackerMaximumAmplification` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. + + .. note:: + + The maximum amplification factor is only considered if the threshold + that can be adjusted by :meth:`.SetAllocTrackerActivationThreshold` + is exceeded. + + .. versionadded:: 3.15 + + :class:`xmlparser` objects have the following attributes: @@ -353,7 +496,7 @@ otherwise stated. ...``). The *doctypeName* is provided exactly as presented. The *systemId* and *publicId* parameters give the system and public identifiers if specified, or ``None`` if omitted. *has_internal_subset* will be true if the document - contains and internal document declaration subset. This requires Expat version + contains an internal document declaration subset. This requires Expat version 1.2 or newer. @@ -502,6 +645,15 @@ otherwise stated. .. method:: xmlparser.ExternalEntityRefHandler(context, base, systemId, publicId) + .. warning:: + + Implementing a handler that accesses local files and/or the network + may create a vulnerability to + `external entity attacks `_ + if :class:`xmlparser` is used with user-provided XML content. + Please reflect on your `threat model `_ + before implementing this handler. + Called for references to external entities. *base* is the current base, as set by a previous call to :meth:`SetBase`. The public and system identifiers, *systemId* and *publicId*, are strings if given; if the public identifier is not @@ -525,9 +677,6 @@ otherwise stated. ExpatError Exceptions --------------------- -.. sectionauthor:: Fred L. Drake, Jr. - - :exc:`ExpatError` exceptions have a number of interesting attributes: @@ -611,14 +760,12 @@ Content Model Descriptions .. module:: xml.parsers.expat.model -.. sectionauthor:: Fred L. Drake, Jr. - Content models are described using nested tuples. Each tuple contains four values: the type, the quantifier, the name, and a tuple of children. Children are simply additional content model descriptions. The values of the first two fields are constants defined in the -:mod:`xml.parsers.expat.model` module. These constants can be collected in two +:mod:`!xml.parsers.expat.model` module. These constants can be collected in two groups: the model type group and the quantifier group. The constants in the model type group are: @@ -692,7 +839,7 @@ Expat error constants .. module:: xml.parsers.expat.errors -The following constants are provided in the :mod:`xml.parsers.expat.errors` +The following constants are provided in the :mod:`!xml.parsers.expat.errors` module. These constants are useful in interpreting some of the attributes of the :exc:`ExpatError` exception objects raised when an error has occurred. Since for backwards compatibility reasons, the constants' value is the error @@ -839,7 +986,7 @@ The ``errors`` module has the following attributes: An operation was requested that requires DTD support to be compiled in, but Expat was configured without DTD support. This should never be reported by a - standard build of the :mod:`xml.parsers.expat` module. + standard build of the :mod:`!xml.parsers.expat` module. .. data:: XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING @@ -949,8 +1096,13 @@ The ``errors`` module has the following attributes: .. rubric:: Footnotes -.. [1] The encoding string included in XML output should conform to the - appropriate standards. For example, "UTF-8" is valid, but "UTF8" is - not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl +.. [1] The encoding string included in XML output should conform to + the appropriate standards. For example, "UTF-8" is valid, but + "UTF8" is not valid in an XML document's declaration, even though + Python accepts it as an encoding name. + See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl and https://www.iana.org/assignments/character-sets/character-sets.xhtml. + +.. _billion laughs: https://en.wikipedia.org/wiki/Billion_laughs_attack +.. |xml-non-root-parser| replace:: :ref:`non-root ` diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index 6dcf06aab002955..f5326aff7236bd6 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`queue` module implements multi-producer, multi-consumer queues. +The :mod:`!queue` module implements multi-producer, multi-consumer queues. It is especially useful in threaded programming when information must be exchanged safely between multiple threads. The :class:`Queue` class in this module implements all the required locking semantics. @@ -30,7 +30,7 @@ In addition, the module implements a "simple" specific implementation provides additional guarantees in exchange for the smaller functionality. -The :mod:`queue` module defines the following classes and exceptions: +The :mod:`!queue` module defines the following classes and exceptions: .. class:: Queue(maxsize=0) @@ -76,6 +76,8 @@ The :mod:`queue` module defines the following classes and exceptions: Constructor for an unbounded :abbr:`FIFO (first-in, first-out)` queue. Simple queues lack advanced functionality such as task tracking. + Simple queues are :ref:`generic ` over the type of their items. + .. versionadded:: 3.7 @@ -256,9 +258,10 @@ until empty or terminated immediately with a hard shutdown. raise :exc:`ShutDown`. If *immediate* is true, the queue is terminated immediately. - The queue is drained to be completely empty. All callers of - :meth:`~Queue.join` are unblocked regardless of the number - of unfinished tasks. Blocked callers of :meth:`~Queue.get` + The queue is drained to be completely empty and the count + of unfinished tasks is reduced by the number of tasks drained. + If unfinished tasks is zero, callers of :meth:`~Queue.join` + are unblocked. Also, blocked callers of :meth:`~Queue.get` are unblocked and will raise :exc:`ShutDown` because the queue is empty. diff --git a/Doc/library/random.rst b/Doc/library/random.rst index b1120b3a4d8eb4e..73a37e189ddf2a8 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -37,7 +37,7 @@ Class :class:`Random` can also be subclassed if you want to use a different basic generator of your own devising: see the documentation on that class for more details. -The :mod:`random` module also provides the :class:`SystemRandom` class which +The :mod:`!random` module also provides the :class:`SystemRandom` class which uses the system function :func:`os.urandom` to generate random numbers from sources provided by the operating system. @@ -78,7 +78,7 @@ Bookkeeping functions instead of the system time (see the :func:`os.urandom` function for details on availability). - If *a* is an int, it is used directly. + If *a* is an int, its absolute value is used directly. With version 2 (the default), a :class:`str`, :class:`bytes`, or :class:`bytearray` object gets converted to an :class:`int` and all of its bits are used. @@ -410,7 +410,7 @@ Alternative Generator .. class:: Random([seed]) Class that implements the default pseudo-random number generator used by the - :mod:`random` module. + :mod:`!random` module. .. versionchanged:: 3.11 Formerly the *seed* could be any hashable object. Now it is limited to: @@ -630,14 +630,16 @@ Recipes ------- These recipes show how to efficiently make random selections -from the combinatoric iterators in the :mod:`itertools` module: +from the combinatoric iterators in the :mod:`itertools` module +or the :pypi:`more-itertools` project: .. testcode:: + import random - def random_product(*args, repeat=1): - "Random selection from itertools.product(*args, **kwds)" - pools = [tuple(pool) for pool in args] * repeat + def random_product(*iterables, repeat=1): + "Random selection from itertools.product(*iterables, repeat=repeat)" + pools = tuple(map(tuple, iterables)) * repeat return tuple(map(random.choice, pools)) def random_permutation(iterable, r=None): @@ -661,6 +663,91 @@ from the combinatoric iterators in the :mod:`itertools` module: indices = sorted(random.choices(range(n), k=r)) return tuple(pool[i] for i in indices) + def random_derangement(iterable): + "Choose a permutation where no element stays in its original position." + seq = tuple(iterable) + if len(seq) < 2: + if not seq: + return () + raise IndexError('No derangments to choose from') + perm = list(range(len(seq))) + start = tuple(perm) + while True: + random.shuffle(perm) + if all(p != q for p, q in zip(start, perm)): + return tuple([seq[i] for i in perm]) + +.. doctest:: + :hide: + + >>> import random + + + >>> random.seed(8675309) + >>> random_product('ABCDEFG', repeat=5) + ('D', 'B', 'E', 'F', 'E') + + + >>> random.seed(8675309) + >>> random_permutation('ABCDEFG') + ('D', 'B', 'E', 'C', 'G', 'A', 'F') + >>> random_permutation('ABCDEFG', 5) + ('A', 'G', 'D', 'C', 'B') + + + >>> random.seed(8675309) + >>> random_combination('ABCDEFG', 7) + ('A', 'B', 'C', 'D', 'E', 'F', 'G') + >>> random_combination('ABCDEFG', 6) + ('A', 'B', 'C', 'D', 'F', 'G') + >>> random_combination('ABCDEFG', 5) + ('A', 'B', 'C', 'E', 'F') + >>> random_combination('ABCDEFG', 4) + ('B', 'C', 'D', 'G') + >>> random_combination('ABCDEFG', 3) + ('B', 'E', 'G') + >>> random_combination('ABCDEFG', 2) + ('E', 'G') + >>> random_combination('ABCDEFG', 1) + ('C',) + >>> random_combination('ABCDEFG', 0) + () + + + >>> random.seed(8675309) + >>> random_combination_with_replacement('ABCDEFG', 7) + ('B', 'C', 'D', 'E', 'E', 'E', 'G') + >>> random_combination_with_replacement('ABCDEFG', 3) + ('A', 'B', 'E') + >>> random_combination_with_replacement('ABCDEFG', 2) + ('A', 'G') + >>> random_combination_with_replacement('ABCDEFG', 1) + ('E',) + >>> random_combination_with_replacement('ABCDEFG', 0) + () + + + >>> random.seed(8675309) + >>> random_derangement('') + () + >>> random_derangement('A') + Traceback (most recent call last): + ... + IndexError: No derangments to choose from + >>> random_derangement('AB') + ('B', 'A') + >>> random_derangement('ABC') + ('C', 'A', 'B') + >>> random_derangement('ABCD') + ('B', 'A', 'D', 'C') + >>> random_derangement('ABCDE') + ('B', 'C', 'A', 'E', 'D') + >>> # Identical inputs treated as distinct + >>> identical = 20 + >>> random_derangement((10, identical, 30, identical)) + (20, 30, 10, 20) + + The default :func:`.random` returns multiples of 2⁻⁵³ in the range *0.0 ≤ x < 1.0*. All such numbers are evenly spaced and are exactly representable as Python floats. However, many other representable diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 75ebbf11c8e47c2..4745c1b98a45543 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -4,9 +4,6 @@ .. module:: re :synopsis: Regular expression operations. -.. moduleauthor:: Fredrik Lundh -.. sectionauthor:: Andrew M. Kuchling - **Source code:** :source:`Lib/re/` -------------- @@ -49,13 +46,13 @@ fine-tuning parameters. .. seealso:: The third-party :pypi:`regex` module, - which has an API compatible with the standard library :mod:`re` module, + which has an API compatible with the standard library :mod:`!re` module, but offers additional functionality and a more thorough Unicode support. .. _re-syntax: -Regular Expression Syntax +Regular expression syntax ------------------------- A regular expression (or RE) specifies a set of strings that matches it; the @@ -208,7 +205,7 @@ The special characters are: *without* establishing any backtracking points. This is the possessive version of the quantifier above. For example, on the 6-character string ``'aaaaaa'``, ``a{3,5}+aa`` - attempt to match 5 ``'a'`` characters, then, requiring 2 more ``'a'``\ s, + attempts to match 5 ``'a'`` characters, then, requiring 2 more ``'a'``\ s, will need more characters than available and thus fail, while ``a{3,5}aa`` will match with ``a{3,5}`` capturing 5, then 4 ``'a'``\ s by backtracking and then the final 2 ``'a'``\ s are matched by the final @@ -720,7 +717,7 @@ three digits in length. .. _contents-of-module-re: -Module Contents +Module contents --------------- The module defines several functions, constants, and an exception. Some of the @@ -836,8 +833,8 @@ Flags will be conditionally ORed with other flags. Example of use as a default value:: - def myfunc(text, flag=re.NOFLAG): - return re.match(text, flag) + def myfunc(pattern, text, flag=re.NOFLAG): + return re.search(pattern, text, flag) .. versionadded:: 3.11 @@ -893,8 +890,8 @@ Functions Compile a regular expression pattern into a :ref:`regular expression object `, which can be used for matching using its - :func:`~Pattern.match`, :func:`~Pattern.search` and other methods, described - below. + :func:`~Pattern.prefixmatch`, + :func:`~Pattern.search`, and other methods, described below. The expression's behaviour can be modified by specifying a *flags* value. Values can be any of the `flags`_ variables, combined using bitwise OR @@ -903,11 +900,11 @@ Functions The sequence :: prog = re.compile(pattern) - result = prog.match(string) + result = prog.search(string) is equivalent to :: - result = re.match(pattern, string) + result = re.search(pattern, string) but using :func:`re.compile` and saving the resulting regular expression object for reuse is more efficient when the expression will be used several @@ -933,15 +930,17 @@ Functions (the ``|`` operator). -.. function:: match(pattern, string, flags=0) +.. function:: prefixmatch(pattern, string, flags=0) If zero or more characters at the beginning of *string* match the regular expression *pattern*, return a corresponding :class:`~re.Match`. Return ``None`` if the string does not match the pattern; note that this is different from a zero-length match. - Note that even in :const:`MULTILINE` mode, :func:`re.match` will only match - at the beginning of the string and not at the beginning of each line. + .. note:: + + Even in :const:`MULTILINE` mode, this will only match at the + beginning of the string and not at the beginning of each line. If you want to locate a match anywhere in *string*, use :func:`search` instead (see also :ref:`search-vs-match`). @@ -950,6 +949,23 @@ Functions Values can be any of the `flags`_ variables, combined using bitwise OR (the ``|`` operator). + This function now has two names and has long been known as + :func:`~re.match`. Use that name when you need to retain compatibility with + older Python versions. + + .. versionadded:: 3.15 + +.. function:: match(pattern, string, flags=0) + + .. soft-deprecated:: 3.15 + :func:`~re.match` has been :term:`soft deprecated` in favor of + the alternate :func:`~re.prefixmatch` name of this API which is + more explicitly descriptive. Use it to better + express intent. The norm in other languages and regular expression + implementations is to use the term *match* to refer to the behavior of + what Python has always called :func:`~re.search`. + See :ref:`prefixmatch-vs-match`. + .. function:: fullmatch(pattern, string, flags=0) @@ -1234,13 +1250,16 @@ Exceptions .. _re-objects: -Regular Expression Objects +Regular expression objects -------------------------- .. class:: Pattern Compiled regular expression object returned by :func:`re.compile`. + Patterns are :ref:`generic ` over the type of string they handle + (:class:`str` or :class:`bytes`). + .. versionchanged:: 3.9 :py:class:`re.Pattern` supports ``[]`` to indicate a Unicode (str) or bytes pattern. See :ref:`types-genericalias`. @@ -1271,24 +1290,44 @@ Regular Expression Objects >>> pattern.search("dog", 1) # No match; search doesn't include the "d" -.. method:: Pattern.match(string[, pos[, endpos]]) +.. method:: Pattern.prefixmatch(string[, pos[, endpos]]) If zero or more characters at the *beginning* of *string* match this regular expression, return a corresponding :class:`~re.Match`. Return ``None`` if the string does not match the pattern; note that this is different from a zero-length match. + Note that even in :const:`MULTILINE` mode, this will only match at the + beginning of the string and not at the beginning of each line. + The optional *pos* and *endpos* parameters have the same meaning as for the :meth:`~Pattern.search` method. :: >>> pattern = re.compile("o") - >>> pattern.match("dog") # No match as "o" is not at the start of "dog". - >>> pattern.match("dog", 1) # Match as "o" is the 2nd character of "dog". + >>> pattern.prefixmatch("dog") # No match as "o" is not at the start of "dog". + >>> pattern.prefixmatch("dog", 1) # Match as "o" is the 2nd character of "dog". If you want to locate a match anywhere in *string*, use :meth:`~Pattern.search` instead (see also :ref:`search-vs-match`). + This method now has two names and has long been known as + :meth:`~Pattern.match`. Use that name when you need to retain compatibility + with older Python versions. + + .. versionadded:: 3.15 + +.. method:: Pattern.match(string[, pos[, endpos]]) + + .. soft-deprecated:: 3.15 + :meth:`~Pattern.match` has been :term:`soft deprecated` in favor of + the alternate :meth:`~Pattern.prefixmatch` name of this API which is + more explicitly descriptive. Use it to + better express intent. The norm in other languages and regular expression + implementations is to use the term *match* to refer to the behavior of + what Python has always called :meth:`~Pattern.search`. + See :ref:`prefixmatch-vs-match`. + .. method:: Pattern.fullmatch(string[, pos[, endpos]]) @@ -1368,7 +1407,7 @@ Regular Expression Objects .. _match-objects: -Match Objects +Match objects ------------- Match objects always have a boolean value of ``True``. @@ -1376,14 +1415,16 @@ Since :meth:`~Pattern.match` and :meth:`~Pattern.search` return ``None`` when there is no match, you can test whether there was a match with a simple ``if`` statement:: - match = re.search(pattern, string) - if match: + if match := re.search(pattern, string): process(match) .. class:: Match Match object returned by successful ``match``\ es and ``search``\ es. + Matches are :ref:`generic ` over the type of string which was + matched (:class:`str` or :class:`bytes`). + .. versionchanged:: 3.9 :py:class:`re.Match` supports ``[]`` to indicate a Unicode (str) or bytes match. See :ref:`types-genericalias`. @@ -1407,23 +1448,23 @@ when there is no match, you can test whether there was a match with a simple result is a single string; if there are multiple arguments, the result is a tuple with one item per argument. Without arguments, *group1* defaults to zero (the whole match is returned). If a *groupN* argument is zero, the corresponding - return value is the entire matching string; if it is in the inclusive range - [1..99], it is the string matching the corresponding parenthesized group. If a - group number is negative or larger than the number of groups defined in the - pattern, an :exc:`IndexError` exception is raised. If a group is contained in a + return value is the entire matching string; if it is a positive integer, it is + the string matching the corresponding parenthesized group. If a group number is + negative or larger than the number of groups defined in the pattern, an + :exc:`IndexError` exception is raised. If a group is contained in a part of the pattern that did not match, the corresponding result is ``None``. If a group is contained in a part of the pattern that matched multiple times, the last match is returned. :: - >>> m = re.match(r"(\w+) (\w+)", "Isaac Newton, physicist") + >>> m = re.search(r"\A(\w+) (\w+)", "Norwegian Blue, pining for the fjords") >>> m.group(0) # The entire match - 'Isaac Newton' + 'Norwegian Blue' >>> m.group(1) # The first parenthesized subgroup. - 'Isaac' + 'Norwegian' >>> m.group(2) # The second parenthesized subgroup. - 'Newton' + 'Blue' >>> m.group(1, 2) # Multiple arguments give us a tuple. - ('Isaac', 'Newton') + ('Norwegian', 'Blue') If the regular expression uses the ``(?P...)`` syntax, the *groupN* arguments may also be strings identifying groups by their group name. If a @@ -1432,23 +1473,23 @@ when there is no match, you can test whether there was a match with a simple A moderately complicated example:: - >>> m = re.match(r"(?P\w+) (?P\w+)", "Malcolm Reynolds") - >>> m.group('first_name') - 'Malcolm' - >>> m.group('last_name') - 'Reynolds' + >>> m = re.search(r"(?P\w+) (?P\w+)", "killer rabbit") + >>> m.group('adjective') + 'killer' + >>> m.group('animal') + 'rabbit' Named groups can also be referred to by their index:: >>> m.group(1) - 'Malcolm' + 'killer' >>> m.group(2) - 'Reynolds' + 'rabbit' If a group matches multiple times, only the last match is accessible:: - >>> m = re.match(r"(..)+", "a1b2c3") # Matches 3 times. - >>> m.group(1) # Returns only the last match. + >>> m = re.search(r"(..)+", "a1b2c3") # Matches 3 times. + >>> m.group(1) # Returns only the last match. 'c3' @@ -1457,21 +1498,21 @@ when there is no match, you can test whether there was a match with a simple This is identical to ``m.group(g)``. This allows easier access to an individual group from a match:: - >>> m = re.match(r"(\w+) (\w+)", "Isaac Newton, physicist") + >>> m = re.search(r"(\w+) (\w+)", "Norwegian Blue, pining for the fjords") >>> m[0] # The entire match - 'Isaac Newton' + 'Norwegian Blue' >>> m[1] # The first parenthesized subgroup. - 'Isaac' + 'Norwegian' >>> m[2] # The second parenthesized subgroup. - 'Newton' + 'Blue' Named groups are supported as well:: - >>> m = re.match(r"(?P\w+) (?P\w+)", "Isaac Newton") - >>> m['first_name'] - 'Isaac' - >>> m['last_name'] - 'Newton' + >>> m = re.search(r"(?P\w+) (?P\w+)", "killer rabbit") + >>> m['adjective'] + 'killer' + >>> m['animal'] + 'rabbit' .. versionadded:: 3.6 @@ -1484,7 +1525,7 @@ when there is no match, you can test whether there was a match with a simple For example:: - >>> m = re.match(r"(\d+)\.(\d+)", "24.1632") + >>> m = re.search(r"(\d+)\.(\d+)", "24.1632") >>> m.groups() ('24', '1632') @@ -1492,7 +1533,7 @@ when there is no match, you can test whether there was a match with a simple might participate in the match. These groups will default to ``None`` unless the *default* argument is given:: - >>> m = re.match(r"(\d+)\.?(\d+)?", "24") + >>> m = re.search(r"(\d+)\.?(\d+)?", "24") >>> m.groups() # Second group defaults to None. ('24', None) >>> m.groups('0') # Now, the second group defaults to '0'. @@ -1505,9 +1546,9 @@ when there is no match, you can test whether there was a match with a simple the subgroup name. The *default* argument is used for groups that did not participate in the match; it defaults to ``None``. For example:: - >>> m = re.match(r"(?P\w+) (?P\w+)", "Malcolm Reynolds") + >>> m = re.search(r"(?P\w+) (?P\w+)", "killer rabbit") >>> m.groupdict() - {'first_name': 'Malcolm', 'last_name': 'Reynolds'} + {'adjective': 'killer', 'animal': 'rabbit'} .. method:: Match.start([group]) @@ -1588,11 +1629,11 @@ when there is no match, you can test whether there was a match with a simple .. _re-examples: -Regular Expression Examples +Regular expression examples --------------------------- -Checking for a Pair +Checking for a pair ^^^^^^^^^^^^^^^^^^^ In this example, we'll use the following helper function to display match @@ -1610,42 +1651,41 @@ representing the card with that value. To see if a given string is a valid hand, one could do the following:: - >>> valid = re.compile(r"^[a2-9tjqk]{5}$") - >>> displaymatch(valid.match("akt5q")) # Valid. + >>> valid_hand_re = re.compile(r"^[a2-9tjqk]{5}$") + >>> displaymatch(valid_hand_re.search("akt5q")) # Valid. "" - >>> displaymatch(valid.match("akt5e")) # Invalid. - >>> displaymatch(valid.match("akt")) # Invalid. - >>> displaymatch(valid.match("727ak")) # Valid. + >>> displaymatch(valid_hand_re.search("akt5e")) # Invalid. + >>> displaymatch(valid_hand_re.search("akt")) # Invalid. + >>> displaymatch(valid_hand_re.search("727ak")) # Valid. "" That last hand, ``"727ak"``, contained a pair, or two of the same valued cards. To match this with a regular expression, one could use backreferences as such:: - >>> pair = re.compile(r".*(.).*\1") - >>> displaymatch(pair.match("717ak")) # Pair of 7s. + >>> pair_re = re.compile(r".*(.).*\1") + >>> displaymatch(pair_re.prefixmatch("717ak")) # Pair of 7s. "" - >>> displaymatch(pair.match("718ak")) # No pairs. - >>> displaymatch(pair.match("354aa")) # Pair of aces. + >>> displaymatch(pair_re.prefixmatch("718ak")) # No pairs. + >>> displaymatch(pair_re.prefixmatch("354aa")) # Pair of aces. "" To find out what card the pair consists of, one could use the :meth:`~Match.group` method of the match object in the following manner:: - >>> pair = re.compile(r".*(.).*\1") - >>> pair.match("717ak").group(1) + >>> pair_re = re.compile(r".*(.).*\1") + >>> pair_re.prefixmatch("717ak").group(1) '7' - # Error because re.match() returns None, which doesn't have a group() method: - >>> pair.match("718ak").group(1) + # Error because prefixmatch() returns None, which doesn't have a group() method: + >>> pair_re.prefixmatch("718ak").group(1) Traceback (most recent call last): File "", line 1, in - re.match(r".*(.).*\1", "718ak").group(1) + pair_re.prefixmatch("718ak").group(1) AttributeError: 'NoneType' object has no attribute 'group' - >>> pair.match("354aa").group(1) + >>> pair_re.prefixmatch("354aa").group(1) 'a' - Simulating scanf() ^^^^^^^^^^^^^^^^^^ @@ -1679,38 +1719,41 @@ expressions. | ``%x``, ``%X`` | ``[-+]?(0[xX])?[\dA-Fa-f]+`` | +--------------------------------+---------------------------------------------+ -To extract the filename and numbers from a string like :: +To extract the filename and numbers from a string like: + +.. code-block:: text /usr/sbin/sendmail - 0 errors, 4 warnings -you would use a :c:func:`!scanf` format like :: +you would use a :c:func:`!scanf` format like: + +.. code-block:: text %s - %d errors, %d warnings -The equivalent regular expression would be :: +The equivalent regular expression would be: + +.. code-block:: text (\S+) - (\d+) errors, (\d+) warnings .. _search-vs-match: -search() vs. match() -^^^^^^^^^^^^^^^^^^^^ - -.. sectionauthor:: Fred L. Drake, Jr. +search() vs. prefixmatch() +^^^^^^^^^^^^^^^^^^^^^^^^^^ Python offers different primitive operations based on regular expressions: -+ :func:`re.match` checks for a match only at the beginning of the string ++ :func:`re.prefixmatch` checks for a match only at the beginning of the string + :func:`re.search` checks for a match anywhere in the string (this is what Perl does by default) + :func:`re.fullmatch` checks for entire string to be a match - For example:: - >>> re.match("c", "abcdef") # No match - >>> re.search("c", "abcdef") # Match + >>> re.prefixmatch("c", "abcdef") # No match + >>> re.search("c", "abcdef") # Match >>> re.fullmatch("p.*n", "python") # Match @@ -1719,21 +1762,54 @@ For example:: Regular expressions beginning with ``'^'`` can be used with :func:`search` to restrict the match at the beginning of the string:: - >>> re.match("c", "abcdef") # No match - >>> re.search("^c", "abcdef") # No match - >>> re.search("^a", "abcdef") # Match + >>> re.prefixmatch("c", "abcdef") # No match + >>> re.search("^c", "abcdef") # No match + >>> re.search("^a", "abcdef") # Match -Note however that in :const:`MULTILINE` mode :func:`match` only matches at the +Note however that in :const:`MULTILINE` mode :func:`prefixmatch` only matches at the beginning of the string, whereas using :func:`search` with a regular expression beginning with ``'^'`` will match at the beginning of each line. :: - >>> re.match("X", "A\nB\nX", re.MULTILINE) # No match + >>> re.prefixmatch("X", "A\nB\nX", re.MULTILINE) # No match >>> re.search("^X", "A\nB\nX", re.MULTILINE) # Match +.. _prefixmatch-vs-match: + +prefixmatch() vs. match() +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Why is the :func:`~re.match` function and method discouraged in +favor of the longer :func:`~re.prefixmatch` spelling? + +Many other languages have gained regex support libraries since regular +expressions were added to Python. However in the most popular of those, they +use the term *match* in their APIs to mean the unanchored behavior provided in +Python by :func:`~re.search`. Thus use of the plain term *match* can be +unclear to those used to other languages when reading or writing code and +not familiar with the Python API's divergence from what otherwise become the +industry norm. + +Quoting from the Zen Of Python (``python3 -m this``): *"Explicit is better than +implicit"*. Anyone reading the name :func:`!prefixmatch` is likely to +understand the intended semantics. When reading :func:`!match` there remains +a seed of doubt about the intended behavior to anyone not already familiar with +this old Python gotcha. + +We **do not** plan to remove the older :func:`!match` name, +as it has been used in code for over 30 years. +It has been :term:`soft deprecated`: +code supporting older versions of Python should continue to use :func:`!match`, +while new code should prefer :func:`!prefixmatch`. + +.. versionadded:: 3.15 + :func:`!prefixmatch` + +.. soft-deprecated:: 3.15 + :func:`!match` -Making a Phonebook +Making a phonebook ^^^^^^^^^^^^^^^^^^ :func:`split` splits a string into a list delimited by the passed pattern. The @@ -1794,7 +1870,7 @@ house number from the street name: ['Heather', 'Albrecht', '548.326.4584', '919', 'Park Place']] -Text Munging +Text munging ^^^^^^^^^^^^ :func:`sub` replaces every occurrence of a pattern with a string or the @@ -1814,7 +1890,7 @@ in each word of a sentence except for the first and last characters:: 'Pofsroser Aodlambelk, plasee reoprt yuor asnebces potlmrpy.' -Finding all Adverbs +Finding all adverbs ^^^^^^^^^^^^^^^^^^^ :func:`findall` matches *all* occurrences of a pattern, not just the first @@ -1827,7 +1903,7 @@ the following manner:: ['carefully', 'quickly'] -Finding all Adverbs and their Positions +Finding all adverbs and their positions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If one wants more information about all matches of a pattern than the matched @@ -1843,7 +1919,7 @@ to find all of the adverbs *and their positions* in some text, they would use 40-47: quickly -Raw String Notation +Raw string notation ^^^^^^^^^^^^^^^^^^^ Raw string notation (``r"text"``) keeps regular expressions sane. Without it, @@ -1851,9 +1927,9 @@ every backslash (``'\'``) in a regular expression would have to be prefixed with another one to escape it. For example, the two following lines of code are functionally identical:: - >>> re.match(r"\W(.)\1\W", " ff ") + >>> re.search(r"\W(.)\1\W", " ff ") - >>> re.match("\\W(.)\\1\\W", " ff ") + >>> re.search("\\W(.)\\1\\W", " ff ") When one wants to match a literal backslash, it must be escaped in the regular @@ -1861,13 +1937,13 @@ expression. With raw string notation, this means ``r"\\"``. Without raw string notation, one must use ``"\\\\"``, making the following lines of code functionally identical:: - >>> re.match(r"\\", r"\\") + >>> re.search(r"\\", r"\\") - >>> re.match("\\\\", r"\\") + >>> re.search("\\\\", r"\\") -Writing a Tokenizer +Writing a tokenizer ^^^^^^^^^^^^^^^^^^^ A `tokenizer or scanner `_ @@ -1883,7 +1959,7 @@ successive matches:: class Token(NamedTuple): type: str - value: str + value: int | float | str line: int column: int diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index f649fce5efc3779..234af8d191e3e35 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -2,14 +2,11 @@ =========================================== .. module:: readline - :platform: Unix :synopsis: GNU readline support for Python. -.. sectionauthor:: Skip Montanaro - -------------- -The :mod:`readline` module defines a number of functions to facilitate +The :mod:`!readline` module defines a number of functions to facilitate completion and reading/writing of history files from the Python interpreter. This module can be used directly, or via the :mod:`rlcompleter` module, which supports completion of Python identifiers at the interactive prompt. Settings @@ -26,11 +23,15 @@ Readline library in general. .. include:: ../includes/wasm-mobile-notavail.rst +.. include:: ../includes/optional-module.rst + +.. availability:: Unix. + .. note:: The underlying Readline library API may be implemented by the ``editline`` (``libedit``) library instead of GNU readline. - On macOS the :mod:`readline` module detects which library is being used + On macOS the :mod:`!readline` module detects which library is being used at run time. The configuration file for ``editline`` is different from that @@ -244,6 +245,15 @@ Startup hooks if Python was compiled for a version of the library that supports it. +.. function:: get_pre_input_hook() + + Get the current pre-input hook function, or ``None`` if no pre-input hook + function has been set. This function only exists if Python was compiled + for a version of the library that supports it. + + .. versionadded:: 3.15 + + .. _readline-completion: Completion @@ -253,7 +263,7 @@ The following functions relate to implementing a custom word completion function. This is typically operated by the Tab key, and can suggest and automatically complete a word being typed. By default, Readline is set up to be used by :mod:`rlcompleter` to complete Python identifiers for -the interactive interpreter. If the :mod:`readline` module is to be used +the interactive interpreter. If the :mod:`!readline` module is to be used with a custom completer, a different set of word delimiters should be set. @@ -322,7 +332,7 @@ with a custom completer, a different set of word delimiters should be set. Example ------- -The following example demonstrates how to use the :mod:`readline` module's +The following example demonstrates how to use the :mod:`!readline` module's history reading and writing functions to automatically load and save a history file named :file:`.python_history` from the user's home directory. The code below would normally be executed automatically during interactive sessions @@ -392,3 +402,9 @@ support history save/restore. :: def save_history(self, histfile): readline.set_history_length(1000) readline.write_history_file(histfile) + +.. note:: + + The new :term:`REPL` introduced in version 3.13 doesn't support readline. + However, readline can still be used by setting the :envvar:`PYTHON_BASIC_REPL` + environment variable. diff --git a/Doc/library/reprlib.rst b/Doc/library/reprlib.rst index 28c7855dfeeef3b..d269d8bbaa55da5 100644 --- a/Doc/library/reprlib.rst +++ b/Doc/library/reprlib.rst @@ -4,8 +4,6 @@ .. module:: reprlib :synopsis: Alternate repr() implementation with size limits. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/reprlib.py` -------------- diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index 0515d205bbca0bb..561b2976ecea22b 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -2,12 +2,8 @@ =============================================== .. module:: resource - :platform: Unix :synopsis: An interface to provide resource usage information on the current process. -.. moduleauthor:: Jeremy Hylton -.. sectionauthor:: Jeremy Hylton - -------------- This module provides basic mechanisms for measuring and controlling system @@ -51,6 +47,20 @@ this module for those platforms. Constant used to represent the limit for an unlimited resource. + .. versionchanged:: 3.15 + It is now always positive. + Previously, it could be negative, such as -1 or -3. + + +.. data:: RLIM_SAVED_CUR +.. data:: RLIM_SAVED_MAX + + Constants used to represent the soft and hard limit values if they + cannot be represented in the ``rlim_t`` value in C. + Can be equal to :data:`RLIM_INFINITY`. + + .. versionadded:: 3.15 + .. function:: getrlimit(resource) @@ -63,12 +73,12 @@ this module for those platforms. Sets new limits of consumption of *resource*. The *limits* argument must be a tuple ``(soft, hard)`` of two integers describing the new limits. A value of - :data:`~resource.RLIM_INFINITY` can be used to request a limit that is + :const:`~resource.RLIM_INFINITY` can be used to request a limit that is unlimited. Raises :exc:`ValueError` if an invalid resource is specified, if the new soft limit exceeds the hard limit, or if a process tries to raise its hard limit. - Specifying a limit of :data:`~resource.RLIM_INFINITY` when the hard or + Specifying a limit of :const:`~resource.RLIM_INFINITY` when the hard or system limit for that resource is not unlimited will result in a :exc:`ValueError`. A process with the effective UID of super-user can request any valid limit value, including unlimited, but :exc:`ValueError` @@ -78,7 +88,7 @@ this module for those platforms. ``setrlimit`` may also raise :exc:`error` if the underlying system call fails. - VxWorks only supports setting :data:`RLIMIT_NOFILE`. + VxWorks only supports setting :const:`RLIMIT_NOFILE`. .. audit-event:: resource.setrlimit resource,limits resource.setrlimit @@ -127,7 +137,7 @@ platform. .. data:: RLIMIT_CPU The maximum amount of processor time (in seconds) that a process can use. If - this limit is exceeded, a :const:`SIGXCPU` signal is sent to the process. (See + this limit is exceeded, a :const:`~signal.SIGXCPU` signal is sent to the process. (See the :mod:`signal` module documentation for information about how to catch this signal and do something useful, e.g. flush open files to disk.) @@ -176,8 +186,9 @@ platform. .. data:: RLIMIT_VMEM The largest area of mapped memory which the process may occupy. + Usually an alias of :const:`RLIMIT_AS`. - .. availability:: FreeBSD >= 11. + .. availability:: Solaris, FreeBSD, NetBSD. .. data:: RLIMIT_AS @@ -230,16 +241,18 @@ platform. .. versionadded:: 3.4 + .. data:: RLIMIT_SBSIZE The maximum size (in bytes) of socket buffer usage for this user. This limits the amount of network memory, and hence the amount of mbufs, that this user may hold at any time. - .. availability:: FreeBSD. + .. availability:: FreeBSD, NetBSD. .. versionadded:: 3.4 + .. data:: RLIMIT_SWAP The maximum size (in bytes) of the swap space that may be reserved or @@ -249,18 +262,20 @@ platform. `tuning(7) `__ for a complete description of this sysctl. - .. availability:: FreeBSD. + .. availability:: FreeBSD >= 8. .. versionadded:: 3.4 + .. data:: RLIMIT_NPTS The maximum number of pseudo-terminals created by this user id. - .. availability:: FreeBSD. + .. availability:: FreeBSD >= 8. .. versionadded:: 3.4 + .. data:: RLIMIT_KQUEUES The maximum number of kqueues this user id is allowed to create. @@ -269,6 +284,46 @@ platform. .. versionadded:: 3.10 + +.. data:: RLIMIT_NTHR + + The maximum number of threads for this user id, not counting the main + and kernel threads. + + .. availability:: NetBSD >= 7.0. + + .. versionadded:: 3.15 + + +.. data:: RLIMIT_PIPEBUF + + The maximum total size of in-kernel buffers for bi-directional pipes/fifos + that this user id is allowed to consume. + + .. availability:: FreeBSD >= 14.2. + + .. versionadded:: 3.15 + + +.. data:: RLIMIT_THREADS + + The maximum number of threads each process can create. + + .. availability:: AIX. + + .. versionadded:: 3.15 + + +.. data:: RLIMIT_UMTXP + + The limit of the number of process-shared Posix thread library objects + allocated by user id. + + .. availability:: FreeBSD >= 11. + + .. versionadded:: 3.15 + + Resource Usage -------------- @@ -297,54 +352,54 @@ These functions are used to retrieve resource usage information: print(getrusage(RUSAGE_SELF)) The fields of the return value each describe how a particular system resource - has been used, e.g. amount of time spent running is user mode or number of times + has been used, e.g. amount of time spent running in user mode or number of times the process was swapped out of main memory. Some values are dependent on the - clock tick internal, e.g. the amount of memory the process is using. + clock tick interval, e.g. the amount of memory the process is using. For backward compatibility, the return value is also accessible as a tuple of 16 elements. - The fields :attr:`ru_utime` and :attr:`ru_stime` of the return value are + The fields :attr:`!ru_utime` and :attr:`!ru_stime` of the return value are floating-point values representing the amount of time spent executing in user mode and the amount of time spent executing in system mode, respectively. The remaining values are integers. Consult the :manpage:`getrusage(2)` man page for detailed information about these values. A brief summary is presented here: - +--------+---------------------+---------------------------------------+ - | Index | Field | Resource | - +========+=====================+=======================================+ - | ``0`` | :attr:`ru_utime` | time in user mode (float seconds) | - +--------+---------------------+---------------------------------------+ - | ``1`` | :attr:`ru_stime` | time in system mode (float seconds) | - +--------+---------------------+---------------------------------------+ - | ``2`` | :attr:`ru_maxrss` | maximum resident set size | - +--------+---------------------+---------------------------------------+ - | ``3`` | :attr:`ru_ixrss` | shared memory size | - +--------+---------------------+---------------------------------------+ - | ``4`` | :attr:`ru_idrss` | unshared memory size | - +--------+---------------------+---------------------------------------+ - | ``5`` | :attr:`ru_isrss` | unshared stack size | - +--------+---------------------+---------------------------------------+ - | ``6`` | :attr:`ru_minflt` | page faults not requiring I/O | - +--------+---------------------+---------------------------------------+ - | ``7`` | :attr:`ru_majflt` | page faults requiring I/O | - +--------+---------------------+---------------------------------------+ - | ``8`` | :attr:`ru_nswap` | number of swap outs | - +--------+---------------------+---------------------------------------+ - | ``9`` | :attr:`ru_inblock` | block input operations | - +--------+---------------------+---------------------------------------+ - | ``10`` | :attr:`ru_oublock` | block output operations | - +--------+---------------------+---------------------------------------+ - | ``11`` | :attr:`ru_msgsnd` | messages sent | - +--------+---------------------+---------------------------------------+ - | ``12`` | :attr:`ru_msgrcv` | messages received | - +--------+---------------------+---------------------------------------+ - | ``13`` | :attr:`ru_nsignals` | signals received | - +--------+---------------------+---------------------------------------+ - | ``14`` | :attr:`ru_nvcsw` | voluntary context switches | - +--------+---------------------+---------------------------------------+ - | ``15`` | :attr:`ru_nivcsw` | involuntary context switches | - +--------+---------------------+---------------------------------------+ + +--------+----------------------+---------------------------------------+ + | Index | Field | Resource | + +========+======================+=======================================+ + | ``0`` | :attr:`!ru_utime` | time in user mode (float seconds) | + +--------+----------------------+---------------------------------------+ + | ``1`` | :attr:`!ru_stime` | time in system mode (float seconds) | + +--------+----------------------+---------------------------------------+ + | ``2`` | :attr:`!ru_maxrss` | maximum resident set size | + +--------+----------------------+---------------------------------------+ + | ``3`` | :attr:`!ru_ixrss` | shared memory size | + +--------+----------------------+---------------------------------------+ + | ``4`` | :attr:`!ru_idrss` | unshared memory size | + +--------+----------------------+---------------------------------------+ + | ``5`` | :attr:`!ru_isrss` | unshared stack size | + +--------+----------------------+---------------------------------------+ + | ``6`` | :attr:`!ru_minflt` | page faults not requiring I/O | + +--------+----------------------+---------------------------------------+ + | ``7`` | :attr:`!ru_majflt` | page faults requiring I/O | + +--------+----------------------+---------------------------------------+ + | ``8`` | :attr:`!ru_nswap` | number of swap outs | + +--------+----------------------+---------------------------------------+ + | ``9`` | :attr:`!ru_inblock` | block input operations | + +--------+----------------------+---------------------------------------+ + | ``10`` | :attr:`!ru_oublock` | block output operations | + +--------+----------------------+---------------------------------------+ + | ``11`` | :attr:`!ru_msgsnd` | messages sent | + +--------+----------------------+---------------------------------------+ + | ``12`` | :attr:`!ru_msgrcv` | messages received | + +--------+----------------------+---------------------------------------+ + | ``13`` | :attr:`!ru_nsignals` | signals received | + +--------+----------------------+---------------------------------------+ + | ``14`` | :attr:`!ru_nvcsw` | voluntary context switches | + +--------+----------------------+---------------------------------------+ + | ``15`` | :attr:`!ru_nivcsw` | involuntary context switches | + +--------+----------------------+---------------------------------------+ This function will raise a :exc:`ValueError` if an invalid *who* parameter is specified. It may also raise :exc:`error` exception in unusual circumstances. diff --git a/Doc/library/rlcompleter.rst b/Doc/library/rlcompleter.rst index 91779feb5250139..2acd1df3c49007a 100644 --- a/Doc/library/rlcompleter.rst +++ b/Doc/library/rlcompleter.rst @@ -4,8 +4,6 @@ .. module:: rlcompleter :synopsis: Python identifier completion, suitable for the GNU readline library. -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/rlcompleter.py` -------------- diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index b07ec6e93f80aba..536b5980f86a817 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -4,13 +4,11 @@ .. module:: runpy :synopsis: Locate and run Python modules without importing them first. -.. moduleauthor:: Nick Coghlan - **Source code:** :source:`Lib/runpy.py` -------------- -The :mod:`runpy` module is used to locate and run Python modules without +The :mod:`!runpy` module is used to locate and run Python modules without importing them first. Its main use is to implement the :option:`-m` command line switch that allows scripts to be located using the Python module namespace rather than the filesystem. @@ -20,11 +18,11 @@ current process, and any side effects (such as cached imports of other modules) will remain in place after the functions have returned. Furthermore, any functions and classes defined by the executed code are not -guaranteed to work correctly after a :mod:`runpy` function has returned. +guaranteed to work correctly after a :mod:`!runpy` function has returned. If that limitation is not acceptable for a given use case, :mod:`importlib` is likely to be a more suitable choice than this module. -The :mod:`runpy` module provides two functions: +The :mod:`!runpy` module provides two functions: .. function:: run_module(mod_name, init_globals=None, run_name=None, alter_sys=False) @@ -50,10 +48,10 @@ The :mod:`runpy` module provides two functions: overridden by :func:`run_module`. The special global variables ``__name__``, ``__spec__``, ``__file__``, - ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals - dictionary before the module code is executed. (Note that this is a - minimal set of variables - other variables may be set implicitly as an - interpreter implementation detail.) + ``__loader__`` and ``__package__`` are set in the globals dictionary before + the module code is executed. (Note that this is a minimal set of variables - + other variables may be set implicitly as an interpreter implementation + detail.) ``__name__`` is set to *run_name* if this optional argument is not :const:`None`, to ``mod_name + '.__main__'`` if the named module is a @@ -63,7 +61,7 @@ The :mod:`runpy` module provides two functions: module (that is, ``__spec__.name`` will always be *mod_name* or ``mod_name + '.__main__'``, never *run_name*). - ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` are + ``__file__``, ``__loader__`` and ``__package__`` are :ref:`set as normal ` based on the module spec. If the argument *alter_sys* is supplied and evaluates to :const:`True`, @@ -98,6 +96,9 @@ The :mod:`runpy` module provides two functions: ``__package__`` are deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. + .. versionchanged:: 3.15 + ``__cached__`` is no longer set. + .. function:: run_path(path_name, init_globals=None, run_name=None) .. index:: @@ -125,23 +126,23 @@ The :mod:`runpy` module provides two functions: overridden by :func:`run_path`. The special global variables ``__name__``, ``__spec__``, ``__file__``, - ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals - dictionary before the module code is executed. (Note that this is a - minimal set of variables - other variables may be set implicitly as an - interpreter implementation detail.) + ``__loader__`` and ``__package__`` are set in the globals dictionary before + the module code is executed. (Note that this is a minimal set of variables - + other variables may be set implicitly as an interpreter implementation + detail.) ``__name__`` is set to *run_name* if this optional argument is not :const:`None` and to ``''`` otherwise. If *file_path* directly references a script file (whether as source or as precompiled byte code), then ``__file__`` will be set to - *file_path*, and ``__spec__``, ``__cached__``, ``__loader__`` and + *file_path*, and ``__spec__``, ``__loader__`` and ``__package__`` will all be set to :const:`None`. If *file_path* is a reference to a valid :data:`sys.path` entry, then ``__spec__`` will be set appropriately for the imported :mod:`__main__` module (that is, ``__spec__.name`` will always be ``__main__``). - ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` will be + ``__file__``, ``__loader__`` and ``__package__`` will be :ref:`set as normal ` based on the module spec. A number of alterations are also made to the :mod:`sys` module. Firstly, @@ -173,6 +174,9 @@ The :mod:`runpy` module provides two functions: The setting of ``__cached__``, ``__loader__``, and ``__package__`` are deprecated. + .. versionchanged:: 3.15 + ``__cached__`` is no longer set. + .. seealso:: :pep:`338` -- Executing modules as scripts diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 517dbe8c3218989..037e27f031d0c82 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -4,15 +4,13 @@ .. module:: sched :synopsis: General purpose event scheduler. -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/sched.py` .. index:: single: event scheduling -------------- -The :mod:`sched` module defines a class which implements a general purpose event +The :mod:`!sched` module defines a class which implements a general purpose event scheduler: .. class:: scheduler(timefunc=time.monotonic, delayfunc=time.sleep) @@ -119,9 +117,11 @@ Scheduler Objects function passed to the constructor) for the next event, then execute it and so on until there are no more scheduled events. - If *blocking* is false executes the scheduled events due to expire soonest - (if any) and then return the deadline of the next scheduled call in the - scheduler (if any). + If *blocking* is false, immediately executes all events in the queue which have + a time value less than or equal to the current *timefunc* value (if any) and + returns the difference between the current *timefunc* value and the time value + of the next scheduled event in the scheduler's event queue. If the queue is + empty, returns ``None``. Either *action* or *delayfunc* can raise an exception. In either case, the scheduler will maintain a consistent state and propagate the exception. If an diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst index 75dafc54d40ca58..3b5b57fb1c21702 100644 --- a/Doc/library/secrets.rst +++ b/Doc/library/secrets.rst @@ -4,8 +4,6 @@ .. module:: secrets :synopsis: Generate secure random numbers for managing secrets. -.. moduleauthor:: Steven D'Aprano -.. sectionauthor:: Steven D'Aprano .. versionadded:: 3.6 .. testsetup:: @@ -17,11 +15,11 @@ ------------- -The :mod:`secrets` module is used for generating cryptographically strong +The :mod:`!secrets` module is used for generating cryptographically strong random numbers suitable for managing data such as passwords, account authentication, security tokens, and related secrets. -In particular, :mod:`secrets` should be used in preference to the +In particular, :mod:`!secrets` should be used in preference to the default pseudo-random number generator in the :mod:`random` module, which is designed for modelling and simulation, not security or cryptography. @@ -33,7 +31,7 @@ is designed for modelling and simulation, not security or cryptography. Random numbers -------------- -The :mod:`secrets` module provides access to the most secure source of +The :mod:`!secrets` module provides access to the most secure source of randomness that your operating system provides. .. class:: SystemRandom @@ -58,43 +56,48 @@ randomness that your operating system provides. Generating tokens ----------------- -The :mod:`secrets` module provides functions for generating secure +The :mod:`!secrets` module provides functions for generating secure tokens, suitable for applications such as password resets, hard-to-guess URLs, and similar. -.. function:: token_bytes([nbytes=None]) +.. function:: token_bytes(nbytes=None) Return a random byte string containing *nbytes* number of bytes. - If *nbytes* is ``None`` or not supplied, a reasonable default is - used. + + If *nbytes* is not specified or ``None``, :const:`DEFAULT_ENTROPY` + is used instead. .. doctest:: - >>> token_bytes(16) #doctest:+SKIP + >>> token_bytes(16) # doctest: +SKIP b'\xebr\x17D*t\xae\xd4\xe3S\xb6\xe2\xebP1\x8b' -.. function:: token_hex([nbytes=None]) +.. function:: token_hex(nbytes=None) Return a random text string, in hexadecimal. The string has *nbytes* - random bytes, each byte converted to two hex digits. If *nbytes* is - ``None`` or not supplied, a reasonable default is used. + random bytes, each byte converted to two hex digits. + + If *nbytes* is not specified or ``None``, :const:`DEFAULT_ENTROPY` + is used instead. .. doctest:: - >>> token_hex(16) #doctest:+SKIP + >>> token_hex(16) # doctest: +SKIP 'f9bf78b9a18ce6d46a0cd2b0b86df9da' -.. function:: token_urlsafe([nbytes=None]) +.. function:: token_urlsafe(nbytes=None) Return a random URL-safe text string, containing *nbytes* random bytes. The text is Base64 encoded, so on average each byte results - in approximately 1.3 characters. If *nbytes* is ``None`` or not - supplied, a reasonable default is used. + in approximately 1.3 characters. + + If *nbytes* is not specified or ``None``, :const:`DEFAULT_ENTROPY` + is used instead. .. doctest:: - >>> token_urlsafe(16) #doctest:+SKIP + >>> token_urlsafe(16) # doctest: +SKIP 'Drmhze6EPcv0fN_81Bj-nA' @@ -107,7 +110,7 @@ tokens need to have sufficient randomness. Unfortunately, what is considered sufficient will necessarily increase as computers get more powerful and able to make more guesses in a shorter period. As of 2015, it is believed that 32 bytes (256 bits) of randomness is sufficient for -the typical use-case expected for the :mod:`secrets` module. +the typical use-case expected for the :mod:`!secrets` module. For those who want to manage their own token length, you can explicitly specify how much randomness is used for tokens by giving an :class:`int` @@ -115,11 +118,13 @@ argument to the various ``token_*`` functions. That argument is taken as the number of bytes of randomness to use. Otherwise, if no argument is provided, or if the argument is ``None``, -the ``token_*`` functions will use a reasonable default instead. +the ``token_*`` functions use :const:`DEFAULT_ENTROPY` instead. -.. note:: +.. data:: DEFAULT_ENTROPY + + Default number of bytes of randomness used by the ``token_*`` functions. - That default is subject to change at any time, including during + The exact value is subject to change at any time, including during maintenance releases. @@ -139,7 +144,7 @@ Other functions Recipes and best practices -------------------------- -This section shows recipes and best practices for using :mod:`secrets` +This section shows recipes and best practices for using :mod:`!secrets` to manage a basic level of security. Generate an eight-character alphanumeric password: diff --git a/Doc/library/select.rst b/Doc/library/select.rst index d2094283d547361..6400005871746a5 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -18,7 +18,7 @@ it was last read. .. note:: The :mod:`selectors` module allows high-level and efficient I/O - multiplexing, built upon the :mod:`select` module primitives. Users are + multiplexing, built upon the :mod:`!select` module primitives. Users are encouraged to use the :mod:`selectors` module instead, unless they want precise control over the OS-level primitives used. @@ -37,7 +37,7 @@ The module defines the following: .. function:: devpoll() - (Only supported on Solaris and derivatives.) Returns a ``/dev/poll`` + Returns a ``/dev/poll`` polling object; see section :ref:`devpoll-objects` below for the methods supported by devpoll objects. @@ -54,15 +54,17 @@ The module defines the following: .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. + .. availability:: Solaris. + .. function:: epoll(sizehint=-1, flags=0) - (Only supported on Linux 2.5.44 and newer.) Return an edge polling object, + Return an edge polling object, which can be used as Edge or Level Triggered interface for I/O events. *sizehint* informs epoll about the expected number of events to be registered. It must be positive, or ``-1`` to use the default. It is only - used on older systems where :c:func:`!epoll_create1` is not available; + used on older systems where :manpage:`epoll_create1(2)` is not available; otherwise it has no effect (though its value is still checked). *flags* is deprecated and completely ignored. However, when supplied, its @@ -89,18 +91,27 @@ The module defines the following: The *flags* parameter. ``select.EPOLL_CLOEXEC`` is used by default now. Use :func:`os.set_inheritable` to make the file descriptor inheritable. + .. versionchanged:: 3.15 + + When CPython is built, this function may be disabled using + :option:`--disable-epoll`. + + .. availability:: Linux >= 2.5.44. + .. function:: poll() - (Not supported by all operating systems.) Returns a polling object, which + Returns a polling object, which supports registering and unregistering file descriptors, and then polling them for I/O events; see section :ref:`poll-objects` below for the methods supported by polling objects. + .. availability:: Unix. + .. function:: kqueue() - (Only supported on BSD.) Returns a kernel queue object; see section + Returns a kernel queue object; see section :ref:`kqueue-objects` below for the methods supported by kqueue objects. The new file descriptor is :ref:`non-inheritable `. @@ -108,14 +119,18 @@ The module defines the following: .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. + .. availability:: BSD, macOS. + .. function:: kevent(ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=0) - (Only supported on BSD.) Returns a kernel event object; see section + Returns a kernel event object; see section :ref:`kevent-objects` below for the methods supported by kevent objects. + .. availability:: BSD, macOS. + -.. function:: select(rlist, wlist, xlist[, timeout]) +.. function:: select(rlist, wlist, xlist, timeout=None) This is a straightforward interface to the Unix :c:func:`!select` system call. The first three arguments are iterables of 'waitable objects': either @@ -129,8 +144,9 @@ The module defines the following: Empty iterables are allowed, but acceptance of three empty iterables is platform-dependent. (It is known to work on Unix but not on Windows.) The - optional *timeout* argument specifies a time-out as a floating-point number - in seconds. When the *timeout* argument is omitted the function blocks until + optional *timeout* argument specifies a time-out in seconds; it may be + a non-integer to specify fractions of seconds. + When the *timeout* argument is omitted or ``None``, the function blocks until at least one file descriptor is ready. A time-out value of zero specifies a poll and never blocks. @@ -164,13 +180,16 @@ The module defines the following: :pep:`475` for the rationale), instead of raising :exc:`InterruptedError`. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. data:: PIPE_BUF The minimum number of bytes which can be written without blocking to a pipe when the pipe has been reported as ready for writing by :func:`~select.select`, :func:`!poll` or another interface in this module. This doesn't apply - to other kind of file-like objects such as sockets. + to other kinds of file-like objects such as sockets. This value is guaranteed by POSIX to be at least 512. @@ -181,7 +200,7 @@ The module defines the following: .. _devpoll-objects: -``/dev/poll`` Polling Objects +``/dev/poll`` polling objects ----------------------------- Solaris and derivatives have ``/dev/poll``. While :c:func:`!select` is @@ -222,7 +241,7 @@ object. implement :meth:`!fileno`, so they can also be used as the argument. *eventmask* is an optional bitmask describing the type of events you want to - check for. The constants are the same that with :c:func:`!poll` + check for. The constants are the same as with :c:func:`!poll` object. The default value is a combination of the constants :const:`POLLIN`, :const:`POLLPRI`, and :const:`POLLOUT`. @@ -237,7 +256,7 @@ object. .. method:: devpoll.modify(fd[, eventmask]) This method does an :meth:`unregister` followed by a - :meth:`register`. It is (a bit) more efficient that doing the same + :meth:`register`. It is (a bit) more efficient than doing the same explicitly. @@ -270,55 +289,58 @@ object. :pep:`475` for the rationale), instead of raising :exc:`InterruptedError`. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. _epoll-objects: -Edge and Level Trigger Polling (epoll) Objects +Edge and level trigger polling (epoll) objects ---------------------------------------------- https://linux.die.net/man/4/epoll - *eventmask* - - +-------------------------+-----------------------------------------------+ - | Constant | Meaning | - +=========================+===============================================+ - | :const:`EPOLLIN` | Available for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLOUT` | Available for write | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLPRI` | Urgent data for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLERR` | Error condition happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLHUP` | Hang up happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | - | | Level Trigger behavior | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | - | | pulled out, the fd is internally disabled | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | - | | associated fd has an event. The default (if | - | | this flag is not set) is to wake all epoll | - | | objects polling on a fd. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | - | | down writing half of connection. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDBAND` | Priority data band can be read. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRBAND` | Priority data may be written. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLMSG` | Ignored. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | - +-------------------------+-----------------------------------------------+ + The *eventmask* is a bit mask using the following constants: + + +-------------------------+------------------------------------------------+ + | Constant | Meaning | + +=========================+================================================+ + | :const:`EPOLLIN` | Available for read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLOUT` | Available for write. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLPRI` | Urgent data for read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLERR` | Error condition happened on the associated fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLHUP` | Hang up happened on the associated fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | + | | Level Trigger behavior. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | + | | pulled out, the fd is internally disabled. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | + | | associated fd has an event. The default (if | + | | this flag is not set) is to wake all epoll | + | | objects polling on an fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | + | | down writing half of connection. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDBAND` | Priority data band can be read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT`. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWRBAND` | Priority data may be written. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLMSG` | Ignored. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | + +-------------------------+------------------------------------------------+ .. versionadded:: 3.6 :const:`EPOLLEXCLUSIVE` was added. It's only supported by Linux Kernel 4.5 @@ -350,12 +372,12 @@ Edge and Level Trigger Polling (epoll) Objects .. method:: epoll.register(fd[, eventmask]) - Register a fd descriptor with the epoll object. + Register a file descriptor *fd* with the epoll object. .. method:: epoll.modify(fd, eventmask) - Modify a registered file descriptor. + Modify a registered file descriptor *fd*. .. method:: epoll.unregister(fd) @@ -368,7 +390,9 @@ Edge and Level Trigger Polling (epoll) Objects .. method:: epoll.poll(timeout=None, maxevents=-1) - Wait for events. timeout in seconds (float) + Wait for events. + If *timeout* is given, it specifies the length of time in seconds + (may be non-integer) which the system will wait for events before returning. .. versionchanged:: 3.5 The function is now retried with a recomputed timeout when interrupted by @@ -376,10 +400,13 @@ Edge and Level Trigger Polling (epoll) Objects :pep:`475` for the rationale), instead of raising :exc:`InterruptedError`. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. _poll-objects: -Polling Objects +Polling objects --------------- The :c:func:`!poll` system call, supported on most Unix systems, provides better @@ -404,24 +431,24 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w :const:`POLLPRI`, and :const:`POLLOUT`, described in the table below. If not specified, the default value used will check for all 3 types of events. - +-------------------+------------------------------------------+ - | Constant | Meaning | - +===================+==========================================+ - | :const:`POLLIN` | There is data to read | - +-------------------+------------------------------------------+ - | :const:`POLLPRI` | There is urgent data to read | - +-------------------+------------------------------------------+ - | :const:`POLLOUT` | Ready for output: writing will not block | - +-------------------+------------------------------------------+ - | :const:`POLLERR` | Error condition of some sort | - +-------------------+------------------------------------------+ - | :const:`POLLHUP` | Hung up | - +-------------------+------------------------------------------+ - | :const:`POLLRDHUP`| Stream socket peer closed connection, or | - | | shut down writing half of connection | - +-------------------+------------------------------------------+ - | :const:`POLLNVAL` | Invalid request: descriptor not open | - +-------------------+------------------------------------------+ + +-------------------+-------------------------------------------+ + | Constant | Meaning | + +===================+===========================================+ + | :const:`POLLIN` | There is data to read. | + +-------------------+-------------------------------------------+ + | :const:`POLLPRI` | There is urgent data to read. | + +-------------------+-------------------------------------------+ + | :const:`POLLOUT` | Ready for output: writing will not block. | + +-------------------+-------------------------------------------+ + | :const:`POLLERR` | Error condition of some sort. | + +-------------------+-------------------------------------------+ + | :const:`POLLHUP` | Hung up. | + +-------------------+-------------------------------------------+ + | :const:`POLLRDHUP`| Stream socket peer closed connection, or | + | | shut down writing half of connection. | + +-------------------+-------------------------------------------+ + | :const:`POLLNVAL` | Invalid request: descriptor not open. | + +-------------------+-------------------------------------------+ Registering a file descriptor that's already registered is not an error, and has the same effect as registering the descriptor exactly once. @@ -464,10 +491,15 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w :pep:`475` for the rationale), instead of raising :exc:`InterruptedError`. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + If ``ppoll()`` function is available, *timeout* has a resolution + of ``1`` ns (``1e-6`` ms) instead of ``1`` ms. + .. _kqueue-objects: -Kqueue Objects +Kqueue objects -------------- .. method:: kqueue.close() @@ -496,7 +528,7 @@ Kqueue Objects - changelist must be an iterable of kevent objects or ``None`` - max_events must be 0 or a positive integer - - timeout in seconds (floats possible); the default is ``None``, + - timeout in seconds (non-integers are possible); the default is ``None``, to wait forever .. versionchanged:: 3.5 @@ -505,10 +537,13 @@ Kqueue Objects :pep:`475` for the rationale), instead of raising :exc:`InterruptedError`. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. _kevent-objects: -Kevent Objects +Kevent objects -------------- https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 @@ -528,66 +563,66 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 | Constant | Meaning | +===========================+=============================================+ | :const:`KQ_FILTER_READ` | Takes a descriptor and returns whenever | - | | there is data available to read | + | | there is data available to read. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_WRITE` | Takes a descriptor and returns whenever | - | | there is data available to write | + | | there is data available to write. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_AIO` | AIO requests | + | :const:`KQ_FILTER_AIO` | AIO requests. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_VNODE` | Returns when one or more of the requested | - | | events watched in *fflag* occurs | + | | events watched in *fflag* occurs. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_PROC` | Watch for events on a process id | + | :const:`KQ_FILTER_PROC` | Watch for events on a process ID. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_NETDEV` | Watch for events on a network device | - | | [not available on macOS] | + | | (not available on macOS). | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_SIGNAL` | Returns whenever the watched signal is | - | | delivered to the process | + | | delivered to the process. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer | + | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer. | +---------------------------+---------------------------------------------+ .. attribute:: kevent.flags Filter action. - +---------------------------+---------------------------------------------+ - | Constant | Meaning | - +===========================+=============================================+ - | :const:`KQ_EV_ADD` | Adds or modifies an event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DELETE` | Removes an event from the queue | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ENABLE` | Permitscontrol() to returns the event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DISABLE` | Disablesevent | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_SYSFLAGS` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_FLAG1` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_EOF` | Filter specific EOF condition | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ERROR` | See return values | - +---------------------------+---------------------------------------------+ + +---------------------------+----------------------------------------------+ + | Constant | Meaning | + +===========================+==============================================+ + | :const:`KQ_EV_ADD` | Adds or modifies an event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_DELETE` | Removes an event from the queue. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ENABLE` | Permits control() to return the event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_DISABLE` | Disables event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_SYSFLAGS` | Internal event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_FLAG1` | Internal event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_EOF` | Filter-specific EOF condition. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ERROR` | See return values. | + +---------------------------+----------------------------------------------+ .. attribute:: kevent.fflags - Filter specific flags. + Filter-specific flags. :const:`KQ_FILTER_READ` and :const:`KQ_FILTER_WRITE` filter flags: +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_LOWAT` | low water mark of a socket buffer | + | :const:`KQ_NOTE_LOWAT` | Low water mark of a socket buffer. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_VNODE` filter flags: @@ -595,19 +630,19 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_DELETE` | *unlink()* was called | + | :const:`KQ_NOTE_DELETE` | *unlink()* was called. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_WRITE` | a write occurred | + | :const:`KQ_NOTE_WRITE` | A write occurred. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXTEND` | the file was extended | + | :const:`KQ_NOTE_EXTEND` | The file was extended. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_ATTRIB` | an attribute was changed | + | :const:`KQ_NOTE_ATTRIB` | An attribute was changed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINK` | the link count has changed | + | :const:`KQ_NOTE_LINK` | The link count has changed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_RENAME` | the file was renamed | + | :const:`KQ_NOTE_RENAME` | The file was renamed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_REVOKE` | access to the file was revoked | + | :const:`KQ_NOTE_REVOKE` | Access to the file was revoked. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_PROC` filter flags: @@ -615,22 +650,22 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_EXIT` | the process has exited | + | :const:`KQ_NOTE_EXIT` | The process has exited. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_FORK` | the process has called *fork()* | + | :const:`KQ_NOTE_FORK` | The process has called *fork()*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXEC` | the process has executed a new process | + | :const:`KQ_NOTE_EXEC` | The process has executed a new process. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PCTRLMASK` | internal filter flag | + | :const:`KQ_NOTE_PCTRLMASK` | Internal filter flag. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PDATAMASK` | internal filter flag | + | :const:`KQ_NOTE_PDATAMASK` | Internal filter flag. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACK` | follow a process across *fork()* | + | :const:`KQ_NOTE_TRACK` | Follow a process across *fork()*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_CHILD` | returned on the child process for | - | | *NOTE_TRACK* | + | :const:`KQ_NOTE_CHILD` | Returned on the child process for | + | | *NOTE_TRACK*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACKERR` | unable to attach to a child | + | :const:`KQ_NOTE_TRACKERR` | Unable to attach to a child. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_NETDEV` filter flags (not available on macOS): @@ -638,19 +673,19 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_LINKUP` | link is up | + | :const:`KQ_NOTE_LINKUP` | Link is up. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKDOWN` | link is down | + | :const:`KQ_NOTE_LINKDOWN` | Link is down. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKINV` | link state is invalid | + | :const:`KQ_NOTE_LINKINV` | Link state is invalid. | +----------------------------+--------------------------------------------+ .. attribute:: kevent.data - Filter specific data. + Filter-specific data. .. attribute:: kevent.udata - User defined value. + User-defined value. diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst index ee556f1f3cecae6..2d523a9d2ea440b 100644 --- a/Doc/library/selectors.rst +++ b/Doc/library/selectors.rst @@ -54,7 +54,7 @@ Classes hierarchy:: In the following, *events* is a bitwise mask indicating which I/O events should -be waited for on a given file object. It can be a combination of the modules +be waited for on a given file object. It can be a combination of the module's constants below: +-----------------------+-----------------------------------------------+ diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index b88fe4157bdc29d..bd3d56f6af595ab 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -42,7 +42,7 @@ lots of shared sub-objects. The keys are ordinary strings. determine which accessed entries are mutable, nor which ones were actually mutated). - By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` + By default, :mod:`!shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` for serializing and deserializing. This can be changed by supplying *serializer* and *deserializer*, respectively. @@ -51,7 +51,7 @@ lots of shared sub-objects. The keys are ordinary strings. :term:`bytes-like object`; the *protocol* value may be ignored by the serializer. - The *deserializer* argument must be callable which takes a serialized object + The *deserializer* argument must be a callable which takes a serialized object given as a :class:`bytes` object and returns the corresponding object. A :exc:`ShelveError` is raised if *serializer* is given but *deserializer* @@ -64,7 +64,7 @@ lots of shared sub-objects. The keys are ordinary strings. .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. - .. versionchanged:: next + .. versionchanged:: 3.15 Accepts custom *serializer* and *deserializer* functions in place of :func:`pickle.dumps` and :func:`pickle.loads`. @@ -81,11 +81,11 @@ lots of shared sub-objects. The keys are ordinary strings. .. warning:: - Because the :mod:`shelve` module is backed by :mod:`pickle`, it is insecure + Because the :mod:`!shelve` module is backed by :mod:`pickle`, it is insecure to load a shelf from an untrusted source. Like with pickle, loading a shelf can execute arbitrary code. -Shelf objects support most of methods and operations supported by dictionaries +Shelf objects support most of the methods and operations supported by dictionaries (except copying, constructors and operators ``|`` and ``|=``). This eases the transition from dictionary based scripts to those requiring persistent storage. @@ -103,7 +103,7 @@ Two additional methods are supported: Calls :meth:`sync` and attempts to shrink space used on disk by removing empty space resulting from deletions. - .. versionadded:: next + .. versionadded:: 3.15 .. method:: Shelf.close() @@ -133,7 +133,7 @@ Restrictions database should be fairly small, and in rare cases key collisions may cause the database to refuse updates. -* The :mod:`shelve` module does not support *concurrent* read/write access to +* The :mod:`!shelve` module does not support *concurrent* read/write access to shelved objects. (Multiple simultaneous read accesses are safe.) When a program has a shelf open for writing, no other program should have it open for reading or writing. Unix file locking can be used to solve this, but this @@ -185,7 +185,7 @@ Restrictions :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. - .. versionchanged:: next + .. versionchanged:: 3.15 Added the *serializer* and *deserializer* parameters. @@ -204,7 +204,7 @@ Restrictions optional *protocol*, *writeback*, *keyencoding*, *serializer* and *deserializer* parameters have the same interpretation as in :func:`~shelve.open`. - .. versionchanged:: next + .. versionchanged:: 3.15 Added the *serializer* and *deserializer* parameters. @@ -220,7 +220,7 @@ Restrictions and *deserializer* parameters have the same interpretation as in :func:`~shelve.open`. - .. versionchanged:: next + .. versionchanged:: 3.15 Added the *serializer* and *deserializer* parameters. @@ -274,7 +274,7 @@ Exceptions The *deserializer* and *serializer* arguments must be given together. - .. versionadded:: next + .. versionadded:: 3.15 .. seealso:: @@ -283,5 +283,5 @@ Exceptions Generic interface to ``dbm``-style databases. Module :mod:`pickle` - Object serialization used by :mod:`shelve`. + Object serialization used by :mod:`!shelve`. diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index a96f0864dc12604..2dfb0246d5d90c0 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -4,11 +4,6 @@ .. module:: shlex :synopsis: Simple lexical analysis for Unix shell-like languages. -.. moduleauthor:: Eric S. Raymond -.. moduleauthor:: Gustavo Niemeyer -.. sectionauthor:: Eric S. Raymond -.. sectionauthor:: Gustavo Niemeyer - **Source code:** :source:`Lib/shlex.py` -------------- @@ -18,7 +13,7 @@ simple syntaxes resembling that of the Unix shell. This will often be useful for writing minilanguages, (for example, in run control files for Python applications) or for parsing quoted strings. -The :mod:`shlex` module defines the following functions: +The :mod:`!shlex` module defines the following functions: .. function:: split(s, comments=False, posix=True) @@ -49,12 +44,15 @@ The :mod:`shlex` module defines the following functions: .. versionadded:: 3.8 -.. function:: quote(s) +.. function:: quote(s, *, force=False) Return a shell-escaped version of the string *s*. The returned value is a string that can safely be used as one token in a shell command line, for cases where you cannot use a list. + If *force* is :const:`True`, then *s* is unconditionally quoted, + even if it is already safe for a shell without being quoted. + .. _shlex-quote-warning: .. warning:: @@ -96,9 +94,24 @@ The :mod:`shlex` module defines the following functions: >>> command ['ls', '-l', 'somefile; rm -rf ~'] + The *force* keyword can be used to produce consistent behavior when + escaping multiple strings: + + >>> from shlex import quote + >>> filenames = ['my first file', 'file2', 'file 3'] + >>> filenames_some_escaped = [quote(f) for f in filenames] + >>> filenames_some_escaped + ["'my first file'", 'file2', "'file 3'"] + >>> filenames_all_escaped = [quote(f, force=True) for f in filenames] + >>> filenames_all_escaped + ["'my first file'", "'file2'", "'file 3'"] + .. versionadded:: 3.3 -The :mod:`shlex` module defines the following class: + .. versionchanged:: next + The *force* keyword was added. + +The :mod:`!shlex` module defines the following class: .. class:: shlex(instream=None, infile=None, posix=False, punctuation_chars=False) @@ -214,7 +227,7 @@ A :class:`~shlex.shlex` instance has the following methods: with the name of the current source file and the ``%d`` with the current input line number (the optional arguments can be used to override these). - This convenience is provided to encourage :mod:`shlex` users to generate error + This convenience is provided to encourage :mod:`!shlex` users to generate error messages in the standard, parseable format understood by Emacs and other Unix tools. @@ -343,7 +356,7 @@ variables which either control lexical analysis or can be used for debugging: Parsing Rules ------------- -When operating in non-POSIX mode, :class:`~shlex.shlex` will try to obey to the +When operating in non-POSIX mode, :class:`~shlex.shlex` will try to obey the following rules. * Quote characters are not recognized within words (``Do"Not"Separate`` is @@ -366,7 +379,7 @@ following rules. * It's not possible to parse empty strings, even if quoted. -When operating in POSIX mode, :class:`~shlex.shlex` will try to obey to the +When operating in POSIX mode, :class:`~shlex.shlex` will try to obey the following parsing rules. * Quotes are stripped out, and do not separate words (``"Do"Not"Separate"`` is @@ -382,7 +395,7 @@ following parsing rules. * Enclosing characters in quotes which are part of :attr:`~shlex.escapedquotes` (e.g. ``'"'``) preserves the literal value of all characters within the quotes, with the exception of the characters - mentioned in :attr:`~shlex.escape`. The escape characters retain its + mentioned in :attr:`~shlex.escape`. The escape characters retain their special meaning only when followed by the quote in use, or the escape character itself. Otherwise the escape character will be considered a normal character. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index dde38498206c467..6febc7a187a15f8 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -4,9 +4,6 @@ .. module:: shutil :synopsis: High-level file operations, including copying. -.. sectionauthor:: Fred L. Drake, Jr. -.. partly based on the docstrings - **Source code:** :source:`Lib/shutil.py` .. index:: @@ -15,7 +12,7 @@ -------------- -The :mod:`shutil` module offers a number of high-level operations on files and +The :mod:`!shutil` module offers a number of high-level operations on files and collections of files. In particular, functions are provided which support file copying and removal. For operations on individual files, see also the :mod:`os` module. @@ -90,6 +87,13 @@ Directory and files operations copy the file more efficiently. See :ref:`shutil-platform-dependent-efficient-copy-operations` section. +.. exception:: SpecialFileError + + This exception is raised when :func:`copyfile` or :func:`copytree` attempt + to copy a named pipe. + + .. versionadded:: 2.7 + .. exception:: SameFileError This exception is raised if source and destination in :func:`copyfile` @@ -381,10 +385,14 @@ Directory and files operations If *dst* already exists but is not a directory, it may be overwritten depending on :func:`os.rename` semantics. - If the destination is on the current filesystem, then :func:`os.rename` is - used. Otherwise, *src* is copied to the destination using *copy_function* - and then removed. In case of symlinks, a new symlink pointing to the target - of *src* will be created as the destination and *src* will be removed. + :func:`os.rename` is preferably used internally when *src* and the destination are on + the same filesystem. In case :func:`os.rename` fails due to :exc:`OSError` + (e.g. the user has write permission to the destination file but not to its parent + directory), this method falls back to using *copy_function*, in which case + *src* is copied to the destination using *copy_function* and then removed. + + In case of symlinks, a new symlink pointing to the target of *src* will be + created in or as the destination, and *src* will be removed. If *copy_function* is given, it must be a callable that takes two arguments, *src* and the destination, and will be used to copy *src* to the destination @@ -508,7 +516,7 @@ Directory and files operations .. exception:: Error - This exception collects exceptions that are raised during a multi-file + Subclass of :exc:`OSError` collecting exceptions raised during a multi-file operation. For :func:`copytree`, the exception argument is a list of 3-tuples (*srcname*, *dstname*, *exception*). @@ -533,10 +541,12 @@ On Solaris :func:`os.sendfile` is used. On Windows :func:`shutil.copyfile` uses a bigger default buffer size (1 MiB instead of 64 KiB) and a :func:`memoryview`-based variant of -:func:`shutil.copyfileobj` is used. +:func:`shutil.copyfileobj` is used, which still reads and writes in a loop. +:func:`shutil.copy2` uses the native ``CopyFile2`` call on Windows, which is the most +efficient method, supports copy-on-write, and preserves metadata. If the fast-copy operation fails and no data was written in the destination -file then shutil will silently fallback on using less efficient +file then shutil will silently fall back to less efficient :func:`copyfileobj` function internally. .. versionchanged:: 3.8 @@ -612,8 +622,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Create an archive file (such as zip or tar) and return its name. - *base_name* is the name of the file to create, including the path, minus - any format-specific extension. + *base_name* is a string or :term:`path-like object` specifying the name of + the file to create, including the path, minus any format-specific extension. *format* is the archive format: one of "zip" (if the :mod:`zlib` module is available), "tar", "gztar" (if the @@ -621,13 +631,14 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. available), "xztar" (if the :mod:`lzma` module is available), or "zstdtar" (if the :mod:`compression.zstd` module is available). - *root_dir* is a directory that will be the root directory of the - archive, all paths in the archive will be relative to it; for example, - we typically chdir into *root_dir* before creating the archive. + *root_dir* is a string or :term:`path-like object` specifying a directory + that will be the root directory of the archive, all paths in the archive + will be relative to it; for example, we typically chdir into *root_dir* + before creating the archive. - *base_dir* is the directory where we start archiving from; - i.e. *base_dir* will be the common prefix of all files and - directories in the archive. *base_dir* must be given relative + *base_dir* is a string or :term:`path-like object` specifying a directory + where we start archiving from; i.e. *base_dir* will be the common prefix of + all files and directories in the archive. *base_dir* must be given relative to *root_dir*. See :ref:`shutil-archiving-example-with-basedir` for how to use *base_dir* and *root_dir* together. @@ -662,12 +673,16 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. This function is now made thread-safe during creation of standard ``.zip`` and tar archives. + .. versionchanged:: 3.15 + Accepts a :term:`path-like object` for *base_name*, *root_dir* and + *base_dir*. + .. function:: get_archive_formats() Return a list of supported formats for archiving. Each element of the returned sequence is a tuple ``(name, description)``. - By default :mod:`shutil` provides these formats: + By default :mod:`!shutil` provides these formats: - *zip*: ZIP file (if the :mod:`zlib` module is available). - *tar*: Uncompressed tar file. Uses POSIX.1-2001 pax format for new archives. @@ -685,7 +700,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Register an archiver for the format *name*. - *function* is the callable that will be used to unpack archives. The callable + *function* is the callable that will be used to create archives. The callable will receive the *base_name* of the file to create, followed by the *base_dir* (which defaults to :data:`os.curdir`) to start archiving from. Further arguments are passed as keyword arguments: *owner*, *group*, @@ -738,8 +753,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Never extract archives from untrusted sources without prior inspection. It is possible that files are created outside of the path specified in - the *extract_dir* argument, e.g. members that have absolute filenames - starting with "/" or filenames with two dots "..". + the *extract_dir* argument, for example, members that have absolute filenames + or filenames with ".." components. Since Python 3.14, the defaults for both built-in formats (zip and tar files) will prevent the most dangerous of such security issues, @@ -784,7 +799,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Each element of the returned sequence is a tuple ``(name, extensions, description)``. - By default :mod:`shutil` provides these formats: + By default :mod:`!shutil` provides these formats: - *zip*: ZIP file (unpacking compressed files works only if the corresponding module is available). @@ -860,7 +875,7 @@ In the final archive, :file:`please_add.txt` should be included, but ... root_dir='tmp/root', ... base_dir='structure/content', ... ) - '/Users/tarek/my_archive.tar' + '/Users/tarek/myarchive.tar' Listing the files in the resulting archive gives us: diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index b0307d3dea11709..12ad45f557e6db5 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -36,7 +36,7 @@ Execution of Python signal handlers A Python signal handler does not get executed inside the low-level (C) signal handler. Instead, the low-level signal handler sets a flag which tells the :term:`virtual machine` to execute the corresponding Python signal handler -at a later point(for example at the next :term:`bytecode` instruction). +at a later point (for example, at the next :term:`bytecode` instruction). This has consequences: * It makes little sense to catch synchronous errors like :const:`SIGFPE` or @@ -68,6 +68,11 @@ the synchronization primitives from the :mod:`threading` module instead. Besides, only the main thread of the main interpreter is allowed to set a new signal handler. +.. warning:: + + Synchronization primitives such as :class:`threading.Lock` should not be used + within signal handlers. Doing so can lead to unexpected deadlocks. + Module contents --------------- @@ -92,13 +97,13 @@ The signal module defines three enums: .. class:: Handlers - :class:`enum.IntEnum` collection the constants :const:`SIG_DFL` and :const:`SIG_IGN`. + :class:`enum.IntEnum` collection of the constants :const:`SIG_DFL` and :const:`SIG_IGN`. .. versionadded:: 3.5 .. class:: Sigmasks - :class:`enum.IntEnum` collection the constants :const:`SIG_BLOCK`, :const:`SIG_UNBLOCK` and :const:`SIG_SETMASK`. + :class:`enum.IntEnum` collection of the constants :const:`SIG_BLOCK`, :const:`SIG_UNBLOCK` and :const:`SIG_SETMASK`. .. availability:: Unix. @@ -108,7 +113,7 @@ The signal module defines three enums: .. versionadded:: 3.5 -The variables defined in the :mod:`signal` module are: +The variables defined in the :mod:`!signal` module are: .. data:: SIG_DFL @@ -205,10 +210,28 @@ The variables defined in the :mod:`signal` module are: .. availability:: Unix. +.. data:: SIGPROF + + Profiling timer expired. + + .. availability:: Unix. + +.. data:: SIGQUIT + + Terminal quit signal. + + .. availability:: Unix. + .. data:: SIGSEGV Segmentation fault: invalid memory reference. +.. data:: SIGSTOP + + Stop executing (cannot be caught or ignored). + + .. availability:: Unix. + .. data:: SIGSTKFLT Stack fault on coprocessor. The Linux kernel does not raise this signal: it @@ -237,18 +260,30 @@ The variables defined in the :mod:`signal` module are: .. availability:: Unix. +.. data:: SIGVTALRM + + Virtual timer expired. + + .. availability:: Unix. + .. data:: SIGWINCH Window resize signal. .. availability:: Unix. +.. data:: SIGXCPU + + CPU time limit exceeded. + + .. availability:: Unix. + .. data:: SIG* All the signal numbers are defined symbolically. For example, the hangup signal is defined as :const:`signal.SIGHUP`; the variable names are identical to the names used in C programs, as found in ````. The Unix man page for - ':c:func:`signal`' lists the existing signals (on some systems this is + '``signal``' lists the existing signals (on some systems this is :manpage:`signal(2)`, on others the list is in :manpage:`signal(7)`). Note that not all systems define the same set of signal names; only those names defined by the system are defined by this module. @@ -322,7 +357,7 @@ The variables defined in the :mod:`signal` module are: .. versionadded:: 3.3 -The :mod:`signal` module defines one exception: +The :mod:`!signal` module defines one exception: .. exception:: ItimerError @@ -336,7 +371,7 @@ The :mod:`signal` module defines one exception: alias of :exc:`OSError`. -The :mod:`signal` module defines the following functions: +The :mod:`!signal` module defines the following functions: .. function:: alarm(time) @@ -478,11 +513,11 @@ The :mod:`signal` module defines the following functions: .. versionadded:: 3.3 -.. function:: setitimer(which, seconds, interval=0.0) +.. function:: setitimer(which, seconds, interval=0) Sets given interval timer (one of :const:`signal.ITIMER_REAL`, :const:`signal.ITIMER_VIRTUAL` or :const:`signal.ITIMER_PROF`) specified - by *which* to fire after *seconds* (float is accepted, different from + by *which* to fire after *seconds* (rounded up to microseconds, different from :func:`alarm`) and after that every *interval* seconds (if *interval* is non-zero). The interval timer specified by *which* can be cleared by setting *seconds* to zero. @@ -493,13 +528,18 @@ The :mod:`signal` module defines the following functions: :const:`signal.ITIMER_VIRTUAL` sends :const:`SIGVTALRM`, and :const:`signal.ITIMER_PROF` will deliver :const:`SIGPROF`. - The old values are returned as a tuple: (delay, interval). + The old values are returned as a two-tuple of floats: + (``delay``, ``interval``). Attempting to pass an invalid interval timer will cause an :exc:`ItimerError`. .. availability:: Unix. + .. versionchanged:: 3.15 + Accepts any real numbers as *seconds* and *interval*, not only integers + or floats. + .. function:: getitimer(which) @@ -639,9 +679,8 @@ The :mod:`signal` module defines the following functions: *sigset*. The return value is an object representing the data contained in the - :c:type:`siginfo_t` structure, namely: :attr:`si_signo`, :attr:`si_code`, - :attr:`si_errno`, :attr:`si_pid`, :attr:`si_uid`, :attr:`si_status`, - :attr:`si_band`. + ``siginfo_t`` structure, namely: ``si_signo``, ``si_code``, + ``si_errno``, ``si_pid``, ``si_uid``, ``si_status``, ``si_band``. .. availability:: Unix. @@ -676,6 +715,9 @@ The :mod:`signal` module defines the following functions: by a signal not in *sigset* and the signal handler does not raise an exception (see :pep:`475` for the rationale). + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. _signal-example: diff --git a/Doc/library/site.rst b/Doc/library/site.rst index e98dd83b60eb600..11a5484c2b13362 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -17,7 +17,7 @@ import can be suppressed using the interpreter's :option:`-S` option. Importing this module normally appends site-specific paths to the module search path and adds :ref:`callables `, including :func:`help` to the built-in -namespace. However, Python startup option :option:`-S` blocks this and this module +namespace. However, Python startup option :option:`-S` blocks this, and this module can be safely imported with no automatic modifications to the module search path or additions to the builtins. To explicitly trigger the usual site-specific additions, call the :func:`main` function. @@ -34,7 +34,7 @@ For the head part, it uses ``sys.prefix`` and ``sys.exec_prefix``; empty heads are skipped. For the tail part, it uses the empty string and then :file:`lib/site-packages` (on Windows) or :file:`lib/python{X.Y[t]}/site-packages` (on Unix and macOS). (The -optional suffix "t" indicates the :term:`free threading` build, and is +optional suffix "t" indicates the :term:`free-threaded build`, and is appended if ``"t"`` is present in the :data:`sys.abiflags` constant.) For each of the distinct head-tail combinations, it sees if it refers to an existing @@ -51,11 +51,11 @@ added path for configuration files. .. versionchanged:: 3.14 - :mod:`site` is no longer responsible for updating :data:`sys.prefix` and + :mod:`!site` is no longer responsible for updating :data:`sys.prefix` and :data:`sys.exec_prefix` on :ref:`sys-path-init-virtual-environments`. This is now done during the :ref:`path initialization `. As a result, under :ref:`sys-path-init-virtual-environments`, :data:`sys.prefix` and - :data:`sys.exec_prefix` no longer depend on the :mod:`site` initialization, + :data:`sys.exec_prefix` no longer depend on the :mod:`!site` initialization, and are therefore unaffected by :option:`-S`. .. _site-virtual-environments-configuration: @@ -64,46 +64,128 @@ When running under a :ref:`virtual environment ` :file:`{name}.start` file exists. + + Errors on individual lines no longer abort processing of the rest of the + file. Each error is reported and the remaining lines continue to be + processed. + +.. deprecated-removed:: 3.15 3.20 + + Decoding :file:`{name}.pth` files in any encoding other than ``utf-8-sig`` + is deprecated in Python 3.15, and support for decoding from the locale + encoding will be removed in Python 3.20. + + ``import`` lines in :file:`{name}.pth` files are deprecated and will be + silently ignored in Python 3.18 and 3.19. In Python 3.20 a warning will be + produced for ``import`` lines in :file:`{name}.pth` files. + + +.. _site-start-files: + +Startup entry points (:file:`.start` files) +------------------------------------------- + +.. versionadded:: 3.15 + +A startup entry point file is a file whose name has the form +:file:`{name}.start` and exists in one of the site-packages directories +described above. Each file specifies entry points to be called during +interpreter startup, using the ``pkg.mod:callable`` syntax understood by +:func:`pkgutil.resolve_name`. + +Each non-blank line that does not begin with ``#`` must contain an entry +point reference in the form ``pkg.mod:callable``. The colon and callable +portion are mandatory. Each callable is invoked with no arguments, and +any return value is discarded. + +:file:`.start` files are processed after all :file:`.pth` path extensions +have been applied to :data:`sys.path`, ensuring that paths are available +before any startup code runs. + +Unlike :data:`sys.path` extensions from :file:`.pth` files, duplicate entry +points are **not** de-duplicated --- if an entry point appears more than once, +it will be called more than once. + +If an exception occurs during resolution or invocation of an entry point, +a traceback is printed to :data:`sys.stderr` and processing continues with +the remaining entry points. + +:file:`.start` files must be encoded in UTF-8. + +:pep:`829` defined the original specification for these features. + +.. note:: + + If a :file:`{name}.start` file exists alongside a :file:`{name}.pth` file + with the same base name, any ``import`` lines in the :file:`.pth` file are + ignored in favor of the entry points in the :file:`.start` file. + +.. note:: + + Executable lines (``import`` lines in :file:`{name}.pth` files and + :file:`{name}.start` file entry points) are always run at Python startup + (unless :option:`-S` is given to disable the ``site.py`` module entirely), + regardless of whether a particular module is actually going to be used. + +.. note:: + + :file:`{name}.start` files invoke :func:`pkgutil.resolve_name` with + ``strict=True``, which requires the full ``pkg.mod:callable`` form. + .. index:: single: package triple: path; configuration; file + +Startup file examples +--------------------- + For example, suppose ``sys.prefix`` and ``sys.exec_prefix`` are set to :file:`/usr/local`. The Python X.Y library is then installed in :file:`/usr/local/lib/python{X.Y}`. Suppose this has a subdirectory :file:`/usr/local/lib/python{X.Y}/site-packages` with three -subsubdirectories, :file:`foo`, :file:`bar` and :file:`spam`, and two path +sub-subdirectories, :file:`foo`, :file:`bar` and :file:`spam`, and two path configuration files, :file:`foo.pth` and :file:`bar.pth`. Assume :file:`foo.pth` contains the following:: @@ -130,38 +212,77 @@ directory precedes the :file:`foo` directory because :file:`bar.pth` comes alphabetically before :file:`foo.pth`; and :file:`spam` is omitted because it is not mentioned in either path configuration file. -:mod:`sitecustomize` --------------------- +Let's say that there is also a :file:`foo.start` file containing the +following:: + + # foo package startup code + + foo.submod:initialize + +Now, after ``sys.path`` has been extended as above, and before Python turns +control over to user code, the ``foo.submod`` module is imported and the +``initialize()`` function from that module is called. + + +.. _site-migration-guide: + +Migrating from ``import`` lines in ``.pth`` files to ``.start`` files +--------------------------------------------------------------------- + +If your package currently ships a :file:`{name}.pth` file, you can keep all +``sys.path`` extension lines unchanged. Only ``import`` lines need to be +migrated. + +To migrate, create a callable (taking zero arguments) within an importable +module in your package. Reference it as a ``pkg.mod:callable`` entry point +in a matching :file:`{name}.start` file. Move everything on your ``import`` +line after the first semi-colon into the ``callable()`` function. + +If your package must straddle older Pythons that do not support :pep:`829` +and newer Pythons that do, change the ``import`` lines in your +:file:`{name}.pth` to use the following form: + +.. code-block:: python + + import pkg.mod; pkg.mod.callable() + +Older Pythons will execute these ``import`` lines, while newer Pythons will +ignore them in favor of the :file:`{name}.start` file. After the straddling +period, remove all ``import`` lines from your :file:`.pth` files. + + +:mod:`!sitecustomize` +--------------------- .. module:: sitecustomize After these path manipulations, an attempt is made to import a module named -:mod:`sitecustomize`, which can perform arbitrary site-specific customizations. +:mod:`!sitecustomize`, which can perform arbitrary site-specific customizations. It is typically created by a system administrator in the site-packages directory. If this import fails with an :exc:`ImportError` or its subclass exception, and the exception's :attr:`~ImportError.name` -attribute equals to ``'sitecustomize'``, +attribute equals ``'sitecustomize'``, it is silently ignored. If Python is started without output streams available, as with :file:`pythonw.exe` on Windows (which is used by default to start IDLE), -attempted output from :mod:`sitecustomize` is ignored. Any other exception +attempted output from :mod:`!sitecustomize` is ignored. Any other exception causes a silent and perhaps mysterious failure of the process. -:mod:`usercustomize` --------------------- +:mod:`!usercustomize` +--------------------- .. module:: usercustomize -After this, an attempt is made to import a module named :mod:`usercustomize`, +After this, an attempt is made to import a module named :mod:`!usercustomize`, which can perform arbitrary user-specific customizations, if :data:`~site.ENABLE_USER_SITE` is true. This file is intended to be created in the user site-packages directory (see below), which is part of ``sys.path`` unless disabled by :option:`-s`. If this import fails with an :exc:`ImportError` or its subclass exception, and the exception's :attr:`~ImportError.name` -attribute equals to ``'usercustomize'``, it is silently ignored. +attribute equals ``'usercustomize'``, it is silently ignored. Note that for some non-Unix systems, ``sys.prefix`` and ``sys.exec_prefix`` are empty, and the path manipulations are skipped; however the import of -:mod:`sitecustomize` and :mod:`usercustomize` is still attempted. +:mod:`sitecustomize` and :mod:`!usercustomize` is still attempted. .. currentmodule:: site @@ -173,7 +294,7 @@ Readline configuration On systems that support :mod:`readline`, this module will also import and configure the :mod:`rlcompleter` module, if Python is started in :ref:`interactive mode ` and without the :option:`-S` option. -The default behavior is enable tab-completion and to use +The default behavior is to enable tab completion and to use :file:`~/.python_history` as the history save file. To disable it, delete (or override) the :data:`sys.__interactivehook__` attribute in your :mod:`sitecustomize` or :mod:`usercustomize` module or your @@ -235,10 +356,97 @@ Module contents This function used to be called unconditionally. +.. function:: makepath(*paths) + + Join *paths* with :func:`os.path.join`, attempt to make the result + absolute with :func:`os.path.abspath`, and return a 2-tuple containing + the absolute path and its case-normalized form as produced by + :func:`os.path.normcase`. If :func:`os.path.abspath` raises + :exc:`OSError`, the joined path is used unchanged for the + case-normalization step. + + The second element of the returned tuple is the form used throughout the + :mod:`!site` module to compare paths on case-insensitive file systems, and + is what populates the ``known_paths`` sets that prevent duplicate + :data:`sys.path` entries in various APIs within this module. + + +.. class:: StartupState(known_paths=None) + + Instances of this class accumulate interpreter startup configuration data + from one or more site directories. They are the preferred interface for + batching the processing of :file:`.pth` and :file:`.start` files across + multiple site directories, so that every :data:`sys.path` extension is + visible before any startup code runs. + + The optional *known_paths* argument is a set of case-normalized paths + (which can be produced by :func:`makepath`) used to prevent duplicate + :data:`sys.path` entries. When ``None`` (the default), the set is built + from the current :data:`sys.path`. :func:`main` implicitly uses an + instance of this class. + + Typical use: + + .. code-block:: python + + state = site.StartupState() + for sitedir in site_dirs: + state.addsitedir(sitedir) + state.process() + + .. versionadded:: 3.15 + + .. method:: addsitedir(sitedir) + + Read the :file:`.pth` and :file:`.start` files in *sitedir* and + record their :data:`sys.path` extensions, deprecated :file:`.pth` + ``import`` lines, and :file:`.start` entry points on this state. + The recorded data is not applied until :meth:`process` is called. + + .. method:: addusersitepackages() + + Add the per-user site-packages directory, if enabled and if it exists. + The directory's startup data is accumulated for later processing by + :meth:`process`. + + .. method:: addsitepackages(prefixes=None) + + Add global site-packages directories, computed from *prefixes* or from + the global :data:`PREFIXES` when *prefixes* is ``None``. Each + directory's startup data is accumulated for later processing by + :meth:`process`. + + .. method:: process() + + Apply the accumulated state by first adding the path extensions to + :data:`sys.path`, then executing the :file:`.start` file entry points + and :file:`.pth` file ``import`` lines (:ref:`deprecated + `). + + This method is not idempotent and must not be called more than once + on the same instance. Doing so will apply the accumulated state + more than once, re-running entry points and ``import`` lines. + + .. function:: addsitedir(sitedir, known_paths=None) - Add a directory to sys.path and process its :file:`.pth` files. Typically - used in :mod:`sitecustomize` or :mod:`usercustomize` (see above). + Add a directory to sys.path and parse the :file:`.pth` and :file:`.start` + files found in that directory. Typically used in :mod:`sitecustomize` or + :mod:`usercustomize` (see above). + + The *known_paths* argument is an optional set of case-normalized paths + used to prevent duplicate :data:`sys.path` entries. When ``None`` (the + default), the set is built from the current :data:`sys.path`. + + For batched processing across multiple site directories, build a + :class:`StartupState` explicitly and call :meth:`StartupState.addsitedir` + on it; that defers :file:`.pth` and :file:`.start` processing until a + single :meth:`StartupState.process` call, ensuring every :data:`sys.path` + extension is visible before any startup code runs. + + .. versionchanged:: 3.15 + + Also processes :file:`.start` files. See :ref:`site-start-files`. .. function:: getsitepackages() @@ -270,12 +478,12 @@ Module contents .. _site-commandline: -Command Line Interface +Command-line interface ---------------------- .. program:: site -The :mod:`site` module also provides a way to get the user directories from the +The :mod:`!site` module also provides a way to get the user directories from the command line: .. code-block:: shell-session @@ -307,5 +515,5 @@ value greater than 2 if there is an error. .. seealso:: * :pep:`370` -- Per user site-packages directory + * :pep:`829` -- Startup entry points and the deprecation of import lines in ``.pth`` files * :ref:`sys-path-init` -- The initialization of :data:`sys.path`. - diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index c5f8516f768a68b..5c97199bc453e85 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -4,8 +4,6 @@ .. module:: smtplib :synopsis: SMTP protocol client (requires sockets). -.. sectionauthor:: Eric S. Raymond - **Source code:** :source:`Lib/smtplib.py` .. index:: @@ -14,7 +12,7 @@ -------------- -The :mod:`smtplib` module defines an SMTP client session object that can be used +The :mod:`!smtplib` module defines an SMTP client session object that can be used to send mail to any internet machine with an SMTP or ESMTP listener daemon. For details of SMTP and ESMTP operation, consult :rfc:`821` (Simple Mail Transfer Protocol) and :rfc:`1869` (SMTP Service Extensions). @@ -24,10 +22,13 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). .. class:: SMTP(host='', port=0, local_hostname=None[, timeout], source_address=None) An :class:`SMTP` instance encapsulates an SMTP connection. It has methods - that support a full repertoire of SMTP and ESMTP operations. If the optional - *host* and *port* parameters are given, the SMTP :meth:`connect` method is - called with those parameters during initialization. If specified, - *local_hostname* is used as the FQDN of the local host in the HELO/EHLO + that support a full repertoire of SMTP and ESMTP operations. + + If the host parameter is set to a truthy value, :meth:`SMTP.connect` is called with + host and port automatically when the object is created; otherwise, :meth:`!connect` must + be called manually. + + If specified, *local_hostname* is used as the FQDN of the local host in the HELO/EHLO command. Otherwise, the local hostname is found using :func:`socket.getfqdn`. If the :meth:`connect` call returns anything other than a success code, an :exc:`SMTPConnectError` is raised. The optional @@ -62,6 +63,10 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). ``smtplib.SMTP.send`` with arguments ``self`` and ``data``, where ``data`` is the bytes about to be sent to the remote host. + .. attribute:: SMTP.default_port + + The default port used for SMTP connections (25). + .. versionchanged:: 3.3 Support for the :keyword:`with` statement was added. @@ -80,15 +85,23 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). An :class:`SMTP_SSL` instance behaves exactly the same as instances of :class:`SMTP`. :class:`SMTP_SSL` should be used for situations where SSL is - required from the beginning of the connection and using :meth:`starttls` is - not appropriate. If *host* is not specified, the local host is used. If - *port* is zero, the standard SMTP-over-SSL port (465) is used. The optional - arguments *local_hostname*, *timeout* and *source_address* have the same + required from the beginning of the connection and using :meth:`SMTP.starttls` is + not appropriate. + + If the host parameter is set to a truthy value, :meth:`SMTP.connect` is called with host + and port automatically when the object is created; otherwise, :meth:`!SMTP.connect` must + be called manually. + + The optional arguments *local_hostname*, *timeout* and *source_address* have the same meaning as they do in the :class:`SMTP` class. *context*, also optional, can contain a :class:`~ssl.SSLContext` and allows configuring various aspects of the secure connection. Please read :ref:`ssl-security` for best practices. + .. attribute:: SMTP_SSL.default_port + + The default port used for SMTP-over-SSL connections (465). + .. versionchanged:: 3.3 *context* was added. @@ -112,7 +125,7 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). The LMTP protocol, which is very similar to ESMTP, is heavily based on the standard SMTP client. It's common to use Unix sockets for LMTP, so our - :meth:`connect` method must support that as well as a regular host:port + :meth:`~SMTP.connect` method must support that as well as a regular host:port server. The optional arguments *local_hostname* and *source_address* have the same meaning as they do in the :class:`SMTP` class. To specify a Unix socket, you must use an absolute path for *host*, starting with a '/'. @@ -147,9 +160,15 @@ A nice selection of exceptions is defined as well: .. exception:: SMTPResponseException Base class for all exceptions that include an SMTP error code. These exceptions - are generated in some instances when the SMTP server returns an error code. The - error code is stored in the :attr:`smtp_code` attribute of the error, and the - :attr:`smtp_error` attribute is set to the error message. + are generated in some instances when the SMTP server returns an error code. + + .. attribute:: smtp_code + + The error code. + + .. attribute:: smtp_error + + The error message. .. exception:: SMTPSenderRefused @@ -161,9 +180,13 @@ A nice selection of exceptions is defined as well: .. exception:: SMTPRecipientsRefused - All recipient addresses refused. The errors for each recipient are accessible - through the attribute :attr:`recipients`, which is a dictionary of exactly the - same sort as :meth:`SMTP.sendmail` returns. + All recipient addresses refused. + + .. attribute:: recipients + + A dictionary of exactly the same sort as returned + by :meth:`SMTP.sendmail` containing the errors for + each recipient. .. exception:: SMTPDataError @@ -213,7 +236,6 @@ SMTP Objects An :class:`SMTP` instance has the following methods: - .. method:: SMTP.set_debuglevel(level) Set the debug output level. A value of 1 or ``True`` for *level* results in @@ -250,6 +272,9 @@ An :class:`SMTP` instance has the following methods: 2-tuple of the response code and message sent by the server in its connection response. + If port is not changed from its default value of 0, the value of the :attr:`default_port` + attribute is used. + .. audit-event:: smtplib.connect self,host,port smtplib.SMTP.connect @@ -327,7 +352,7 @@ An :class:`SMTP` instance has the following methods: :exc:`SMTPException` No suitable authentication method was found. - Each of the authentication methods supported by :mod:`smtplib` are tried in + Each of the authentication methods supported by :mod:`!smtplib` are tried in turn if they are advertised as supported by the server. See :meth:`auth` for a list of supported authentication methods. *initial_response_ok* is passed through to :meth:`auth`. @@ -379,7 +404,7 @@ An :class:`SMTP` instance has the following methods: call the :meth:`login` method, which will try each of the above mechanisms in turn, in the order listed. ``auth`` is exposed to facilitate the implementation of authentication methods not (or not yet) supported - directly by :mod:`smtplib`. + directly by :mod:`!smtplib`. .. versionadded:: 3.5 @@ -417,7 +442,7 @@ An :class:`SMTP` instance has the following methods: .. versionchanged:: 3.4 The method now supports hostname check with - :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see + :attr:`ssl.SSLContext.check_hostname` and *Server Name Indicator* (see :const:`~ssl.HAS_SNI`). .. versionchanged:: 3.5 @@ -431,11 +456,13 @@ An :class:`SMTP` instance has the following methods: Send mail. The required arguments are an :rfc:`822` from-address string, a list of :rfc:`822` to-address strings (a bare string will be treated as a list with 1 address), and a message string. The caller may pass a list of ESMTP options - (such as ``8bitmime``) to be used in ``MAIL FROM`` commands as *mail_options*. + (such as ``"8bitmime"``) to be used in ``MAIL FROM`` commands as *mail_options*. ESMTP options (such as ``DSN`` commands) that should be used with all ``RCPT`` - commands can be passed as *rcpt_options*. (If you need to use different ESMTP + commands can be passed as *rcpt_options*. Each option should be passed as a string + containing the full text of the option, including any potential key + (for instance, ``"NOTIFY=SUCCESS,FAILURE"``). (If you need to use different ESMTP options to different recipients you have to use the low-level methods such as - :meth:`mail`, :meth:`rcpt` and :meth:`data` to send the message.) + :meth:`!mail`, :meth:`!rcpt` and :meth:`!data` to send the message.) .. note:: @@ -467,10 +494,7 @@ An :class:`SMTP` instance has the following methods: This method may raise the following exceptions: :exc:`SMTPRecipientsRefused` - All recipients were refused. Nobody got the mail. The :attr:`recipients` - attribute of the exception object is a dictionary with information about the - refused recipients (like the one returned when at least one recipient was - accepted). + All recipients were refused. Nobody got the mail. :exc:`SMTPHeloError` The server didn't reply properly to the ``HELO`` greeting. @@ -546,6 +570,30 @@ Low-level methods corresponding to the standard SMTP/ESMTP commands ``HELP``, Normally these do not need to be called directly, so they are not documented here. For details, consult the module code. +Additionally, an SMTP instance has the following attributes: + + +.. attribute:: SMTP.helo_resp + + The response to the ``HELO`` command, see :meth:`helo`. + + +.. attribute:: SMTP.ehlo_resp + + The response to the ``EHLO`` command, see :meth:`ehlo`. + + +.. attribute:: SMTP.does_esmtp + + A boolean value indicating whether the server supports ESMTP, see + :meth:`ehlo`. + + +.. attribute:: SMTP.esmtp_features + + A dictionary of the names of SMTP service extensions supported by the server, + see :meth:`ehlo`. + .. _smtp-example: diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index bc89a3228f0ed9e..836aa91bb0885b1 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -39,6 +39,8 @@ is implicit on send operations. A TLS/SSL wrapper for socket objects. +.. _socket-addresses: + Socket families --------------- @@ -83,7 +85,7 @@ created. Socket addresses are represented as follows: - For :const:`AF_INET6` address family, a four-tuple ``(host, port, flowinfo, scope_id)`` is used, where *flowinfo* and *scope_id* represent the ``sin6_flowinfo`` and ``sin6_scope_id`` members in :const:`struct sockaddr_in6` in C. For - :mod:`socket` module methods, *flowinfo* and *scope_id* can be omitted just for + :mod:`!socket` module methods, *flowinfo* and *scope_id* can be omitted just for backward compatibility. Note, however, omission of *scope_id* can cause problems in manipulating scoped IPv6 addresses. @@ -118,10 +120,10 @@ created. Socket addresses are represented as follows: ``'can0'``. The network interface name ``''`` can be used to receive packets from all network interfaces of this family. - - :const:`CAN_ISOTP` protocol require a tuple ``(interface, rx_addr, tx_addr)`` + - :const:`CAN_ISOTP` protocol requires a tuple ``(interface, rx_addr, tx_addr)`` where both additional parameters are unsigned long integer that represent a CAN identifier (standard or extended). - - :const:`CAN_J1939` protocol require a tuple ``(interface, name, pgn, addr)`` + - :const:`CAN_J1939` protocol requires a tuple ``(interface, name, pgn, addr)`` where additional parameters are 64-bit unsigned integer representing the ECU name, a 32-bit unsigned integer representing the Parameter Group Number (PGN), and an 8-bit integer representing the address. @@ -302,7 +304,7 @@ generalization of this based on timeouts is supported through Module contents --------------- -The module :mod:`socket` exports the following elements. +The module :mod:`!socket` exports the following elements. Exceptions @@ -482,6 +484,10 @@ The AF_* and SOCK_* constants are now :class:`AddressFamily` and .. versionchanged:: 3.14 Added support for ``TCP_QUICKACK`` on Windows platforms when available. + .. versionchanged:: 3.15 + ``IPV6_HDRINCL`` was added. + Added support for ``SO_PASSRIGHTS`` on Linux platforms when available. + .. data:: AF_CAN PF_CAN @@ -836,71 +842,14 @@ Creating sockets The following functions all create :ref:`socket objects `. -.. class:: socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None) - - Create a new socket using the given address family, socket type and protocol - number. The address family should be :const:`AF_INET` (the default), - :const:`AF_INET6`, :const:`AF_UNIX`, :const:`AF_CAN`, :const:`AF_PACKET`, - or :const:`AF_RDS`. The socket type should be :const:`SOCK_STREAM` (the - default), :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other - ``SOCK_`` constants. The protocol number is usually zero and may be omitted - or in the case where the address family is :const:`AF_CAN` the protocol - should be one of :const:`CAN_RAW`, :const:`CAN_BCM`, :const:`CAN_ISOTP` or - :const:`CAN_J1939`. - - If *fileno* is specified, the values for *family*, *type*, and *proto* are - auto-detected from the specified file descriptor. Auto-detection can be - overruled by calling the function with explicit *family*, *type*, or *proto* - arguments. This only affects how Python represents e.g. the return value - of :meth:`socket.getpeername` but not the actual OS resource. Unlike - :func:`socket.fromfd`, *fileno* will return the same socket and not a - duplicate. This may help close a detached socket using - :meth:`socket.close`. - - The newly created socket is :ref:`non-inheritable `. - - .. audit-event:: socket.__new__ self,family,type,protocol socket.socket - - .. versionchanged:: 3.3 - The AF_CAN family was added. - The AF_RDS family was added. - - .. versionchanged:: 3.4 - The CAN_BCM protocol was added. - - .. versionchanged:: 3.4 - The returned socket is now non-inheritable. - - .. versionchanged:: 3.7 - The CAN_ISOTP protocol was added. - - .. versionchanged:: 3.7 - When :const:`SOCK_NONBLOCK` or :const:`SOCK_CLOEXEC` - bit flags are applied to *type* they are cleared, and - :attr:`socket.type` will not reflect them. They are still passed - to the underlying system ``socket()`` call. Therefore, - - :: - - sock = socket.socket( - socket.AF_INET, - socket.SOCK_STREAM | socket.SOCK_NONBLOCK) - - will still create a non-blocking socket on OSes that support - ``SOCK_NONBLOCK``, but ``sock.type`` will be set to - ``socket.SOCK_STREAM``. - - .. versionchanged:: 3.9 - The CAN_J1939 protocol was added. - - .. versionchanged:: 3.10 - The IPPROTO_MPTCP protocol was added. +The :class:`socket ` class constructor creates a new socket +directly; see :ref:`socket-objects` for its parameters and full description. .. function:: socketpair([family[, type[, proto]]]) Build a pair of connected socket objects using the given address family, socket type, and protocol number. Address family, socket type, and protocol number are - as for the :func:`~socket.socket` function above. The default family is :const:`AF_UNIX` + as for the :func:`~socket.socket` function. The default family is :const:`AF_UNIX` if defined on the platform; otherwise, the default is :const:`AF_INET`. The newly created sockets are :ref:`non-inheritable `. @@ -996,8 +945,8 @@ The following functions all create :ref:`socket objects `. Duplicate the file descriptor *fd* (an integer as returned by a file object's :meth:`~io.IOBase.fileno` method) and build a socket object from the result. Address - family, socket type and protocol number are as for the :func:`~socket.socket` function - above. The file descriptor should refer to a socket, but this is not checked --- + family, socket type and protocol number are as for the :func:`~socket.socket` function. + The file descriptor should refer to a socket, but this is not checked --- subsequent operations on the object may fail if the file descriptor is invalid. This function is rarely needed, but can be used to get or set socket options on a socket passed to a program as standard input or output (such as a server @@ -1019,22 +968,16 @@ The following functions all create :ref:`socket objects `. .. versionadded:: 3.3 -.. data:: SocketType - - This is a Python type object that represents the socket object type. It is the - same as ``type(socket(...))``. - - Other functions ''''''''''''''' -The :mod:`socket` module also offers various network-related services: +The :mod:`!socket` module also offers various network-related services: .. function:: close(fd) Close a socket file descriptor. This is like :func:`os.close`, but for - sockets. On some platforms (most noticeable Windows) :func:`os.close` + sockets. On some platforms (most notably Windows) :func:`os.close` does not work for socket file descriptors. .. versionadded:: 3.7 @@ -1069,10 +1012,16 @@ The :mod:`socket` module also offers various network-related services: a string representing the canonical name of the *host* if :const:`AI_CANONNAME` is part of the *flags* argument; else *canonname* will be empty. *sockaddr* is a tuple describing a socket address, whose - format depends on the returned *family* (a ``(address, port)`` 2-tuple for - :const:`AF_INET`, a ``(address, port, flowinfo, scope_id)`` 4-tuple for - :const:`AF_INET6`), and is meant to be passed to the :meth:`socket.connect` - method. + format depends on the returned *family* and flags Python was compiled with, + and is meant to be passed to the :meth:`socket.connect` method. + + *sockaddr* can be one of the following: + + * a ``(address, port)`` 2-tuple for :const:`AF_INET` + * a ``(address, port, flowinfo, scope_id)`` 4-tuple for :const:`AF_INET6` if + Python was compiled with ``--enable-ipv6`` (the default) + * a 2-tuple containing raw data for :const:`AF_INET6` if Python was + compiled with ``--disable-ipv6`` .. note:: @@ -1407,11 +1356,14 @@ The :mod:`socket` module also offers various network-related services: .. function:: setdefaulttimeout(timeout) - Set the default timeout in seconds (float) for new socket objects. When + Set the default timeout in seconds (real number) for new socket objects. When the socket module is first imported, the default is ``None``. See :meth:`~socket.settimeout` for possible values and their respective meanings. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. + .. function:: sethostname(name) @@ -1523,641 +1475,709 @@ The :mod:`socket` module also offers various network-related services: Socket Objects -------------- -Socket objects have the following methods. Except for -:meth:`~socket.makefile`, these correspond to Unix system calls applicable -to sockets. +.. class:: socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None) -.. versionchanged:: 3.2 - Support for the :term:`context manager` protocol was added. Exiting the - context manager is equivalent to calling :meth:`~socket.close`. + Create a new socket using the given address family, socket type and protocol + number. The address family should be :const:`AF_INET` (the default), + :const:`AF_INET6`, :const:`AF_UNIX`, :const:`AF_CAN`, :const:`AF_PACKET`, + or :const:`AF_RDS`. The socket type should be :const:`SOCK_STREAM` (the + default), :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other + ``SOCK_`` constants. The protocol number is usually zero and may be omitted + or in the case where the address family is :const:`AF_CAN` the protocol + should be one of :const:`CAN_RAW`, :const:`CAN_BCM`, :const:`CAN_ISOTP` or + :const:`CAN_J1939`. + If *fileno* is specified, the values for *family*, *type*, and *proto* are + auto-detected from the specified file descriptor. Auto-detection can be + overruled by calling the function with explicit *family*, *type*, or *proto* + arguments. This only affects how Python represents e.g. the return value + of :meth:`socket.getpeername` but not the actual OS resource. Unlike + :func:`socket.fromfd`, *fileno* will return the same socket and not a + duplicate. This may help close a detached socket using + :meth:`socket.close`. -.. method:: socket.accept() + The newly created socket is :ref:`non-inheritable `. - Accept a connection. The socket must be bound to an address and listening for - connections. The return value is a pair ``(conn, address)`` where *conn* is a - *new* socket object usable to send and receive data on the connection, and - *address* is the address bound to the socket on the other end of the connection. + .. audit-event:: socket.__new__ self,family,type,protocol socket.socket - The newly created socket is :ref:`non-inheritable `. + .. versionchanged:: 3.3 + The AF_CAN family was added. + The AF_RDS family was added. .. versionchanged:: 3.4 - The socket is now non-inheritable. + The CAN_BCM protocol was added. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + .. versionchanged:: 3.4 + The returned socket is now non-inheritable. + .. versionchanged:: 3.7 + The CAN_ISOTP protocol was added. -.. method:: socket.bind(address) + .. versionchanged:: 3.7 + When :const:`SOCK_NONBLOCK` or :const:`SOCK_CLOEXEC` + bit flags are applied to *type* they are cleared, and + :attr:`socket.type` will not reflect them. They are still passed + to the underlying system ``socket()`` call. Therefore, - Bind the socket to *address*. The socket must not already be bound. (The format - of *address* depends on the address family --- see above.) + :: - .. audit-event:: socket.bind self,address socket.socket.bind + sock = socket.socket( + socket.AF_INET, + socket.SOCK_STREAM | socket.SOCK_NONBLOCK) - .. availability:: not WASI. + will still create a non-blocking socket on OSes that support + ``SOCK_NONBLOCK``, but ``sock.type`` will be set to + ``socket.SOCK_STREAM``. + .. versionchanged:: 3.9 + The CAN_J1939 protocol was added. -.. method:: socket.close() + .. versionchanged:: 3.10 + The IPPROTO_MPTCP protocol was added. - Mark the socket closed. The underlying system resource (e.g. a file - descriptor) is also closed when all file objects from :meth:`makefile` - are closed. Once that happens, all future operations on the socket - object will fail. The remote end will receive no more data (after - queued data is flushed). + Socket objects have the following methods. Except for + :meth:`~socket.makefile`, these correspond to Unix system calls applicable + to sockets. - Sockets are automatically closed when they are garbage-collected, but - it is recommended to :meth:`close` them explicitly, or to use a - :keyword:`with` statement around them. + .. versionchanged:: 3.2 + Support for the :term:`context manager` protocol was added. Exiting the + context manager is equivalent to calling :meth:`~socket.close`. - .. versionchanged:: 3.6 - :exc:`OSError` is now raised if an error occurs when the underlying - :c:func:`close` call is made. - .. note:: + .. method:: accept() - :meth:`close` releases the resource associated with a connection but - does not necessarily close the connection immediately. If you want - to close the connection in a timely fashion, call :meth:`shutdown` - before :meth:`close`. + Accept a connection. The socket must be bound to an address and listening for + connections. The return value is a pair ``(conn, address)`` where *conn* is a + *new* socket object usable to send and receive data on the connection, and + *address* is the address bound to the socket on the other end of the connection. + The newly created socket is :ref:`non-inheritable `. -.. method:: socket.connect(address) + .. versionchanged:: 3.4 + The socket is now non-inheritable. - Connect to a remote socket at *address*. (The format of *address* depends on the - address family --- see above.) + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - If the connection is interrupted by a signal, the method waits until the - connection completes, or raise a :exc:`TimeoutError` on timeout, if the - signal handler doesn't raise an exception and the socket is blocking or has - a timeout. For non-blocking sockets, the method raises an - :exc:`InterruptedError` exception if the connection is interrupted by a - signal (or the exception raised by the signal handler). - .. audit-event:: socket.connect self,address socket.socket.connect + .. method:: bind(address) - .. versionchanged:: 3.5 - The method now waits until the connection completes instead of raising an + Bind the socket to *address*. The socket must not already be bound. The format + of *address* depends on the address family --- see :ref:`socket-addresses`. + + .. audit-event:: socket.bind self,address socket.socket.bind + + .. availability:: not WASI. + + + .. method:: close() + + Mark the socket closed. The underlying system resource (e.g. a file + descriptor) is also closed when all file objects from :meth:`makefile` + are closed. Once that happens, all future operations on the socket + object will fail. The remote end will receive no more data (after + queued data is flushed). + + Sockets are automatically closed when they are garbage-collected, but + it is recommended to :meth:`close` them explicitly, or to use a + :keyword:`with` statement around them. + + .. versionchanged:: 3.6 + :exc:`OSError` is now raised if an error occurs when the underlying + :c:func:`!close` call is made. + + .. note:: + + :meth:`close` releases the resource associated with a connection but + does not necessarily close the connection immediately. If you want + to close the connection in a timely fashion, call :meth:`shutdown` + before :meth:`close`. + + + .. method:: connect(address) + + Connect to a remote socket at *address*. The format of *address* depends on the + address family --- see :ref:`socket-addresses`. + + If the connection is interrupted by a signal, the method waits until the + connection completes, or raises a :exc:`TimeoutError` on timeout, if the + signal handler doesn't raise an exception and the socket is blocking or has + a timeout. For non-blocking sockets, the method raises an :exc:`InterruptedError` exception if the connection is interrupted by a - signal, the signal handler doesn't raise an exception and the socket is - blocking or has a timeout (see the :pep:`475` for the rationale). + signal (or the exception raised by the signal handler). - .. availability:: not WASI. + .. audit-event:: socket.connect self,address socket.socket.connect + .. versionchanged:: 3.5 + The method now waits until the connection completes instead of raising an + :exc:`InterruptedError` exception if the connection is interrupted by a + signal, the signal handler doesn't raise an exception and the socket is + blocking or has a timeout (see the :pep:`475` for the rationale). -.. method:: socket.connect_ex(address) + .. availability:: not WASI. - Like ``connect(address)``, but return an error indicator instead of raising an - exception for errors returned by the C-level :c:func:`connect` call (other - problems, such as "host not found," can still raise exceptions). The error - indicator is ``0`` if the operation succeeded, otherwise the value of the - :c:data:`errno` variable. This is useful to support, for example, asynchronous - connects. - .. audit-event:: socket.connect self,address socket.socket.connect_ex + .. method:: connect_ex(address) - .. availability:: not WASI. + Like ``connect(address)``, but return an error indicator instead of raising an + exception for errors returned by the C-level :c:func:`!connect` call (other + problems, such as "host not found," can still raise exceptions). The error + indicator is ``0`` if the operation succeeded, otherwise the value of the + :c:data:`errno` variable. This is useful to support, for example, asynchronous + connects. -.. method:: socket.detach() + .. audit-event:: socket.connect self,address socket.socket.connect_ex - Put the socket object into closed state without actually closing the - underlying file descriptor. The file descriptor is returned, and can - be reused for other purposes. + .. availability:: not WASI. - .. versionadded:: 3.2 + .. method:: detach() + Put the socket object into closed state without actually closing the + underlying file descriptor. The file descriptor is returned, and can + be reused for other purposes. -.. method:: socket.dup() + .. versionadded:: 3.2 - Duplicate the socket. - The newly created socket is :ref:`non-inheritable `. + .. method:: dup() - .. versionchanged:: 3.4 - The socket is now non-inheritable. + Duplicate the socket. - .. availability:: not WASI. + The newly created socket is :ref:`non-inheritable `. + .. versionchanged:: 3.4 + The socket is now non-inheritable. -.. method:: socket.fileno() + .. availability:: not WASI. - Return the socket's file descriptor (a small integer), or -1 on failure. This - is useful with :func:`select.select`. - Under Windows the small integer returned by this method cannot be used where a - file descriptor can be used (such as :func:`os.fdopen`). Unix does not have - this limitation. + .. method:: fileno() -.. method:: socket.get_inheritable() + Return the socket's file descriptor (a small integer), or -1 on failure. This + is useful with :func:`select.select`. - Get the :ref:`inheritable flag ` of the socket's file - descriptor or socket's handle: ``True`` if the socket can be inherited in - child processes, ``False`` if it cannot. + Under Windows the small integer returned by this method cannot be used where a + file descriptor can be used (such as :func:`os.fdopen`). Unix does not have + this limitation. - .. versionadded:: 3.4 + .. method:: get_inheritable() + Get the :ref:`inheritable flag ` of the socket's file + descriptor or socket's handle: ``True`` if the socket can be inherited in + child processes, ``False`` if it cannot. -.. method:: socket.getpeername() + .. versionadded:: 3.4 - Return the remote address to which the socket is connected. This is useful to - find out the port number of a remote IPv4/v6 socket, for instance. (The format - of the address returned depends on the address family --- see above.) On some - systems this function is not supported. + .. method:: getpeername() -.. method:: socket.getsockname() + Return the remote address to which the socket is connected. This is useful to + find out the port number of a remote IPv4/v6 socket, for instance. The format + of the address returned depends on the address family --- see :ref:`socket-addresses`. + On some systems this function is not supported. - Return the socket's own address. This is useful to find out the port number of - an IPv4/v6 socket, for instance. (The format of the address returned depends on - the address family --- see above.) + .. method:: getsockname() -.. method:: socket.getsockopt(level, optname[, buflen]) + Return the socket's own address. This is useful to find out the port number of + an IPv4/v6 socket, for instance. The format of the address returned depends on + the address family --- see :ref:`socket-addresses`. - Return the value of the given socket option (see the Unix man page - :manpage:`getsockopt(2)`). The needed symbolic constants (:ref:`SO_\* etc. `) - are defined in this module. If *buflen* is absent, an integer option is assumed - and its integer value is returned by the function. If *buflen* is present, it - specifies the maximum length of the buffer used to receive the option in, and - this buffer is returned as a bytes object. It is up to the caller to decode the - contents of the buffer (see the optional built-in module :mod:`struct` for a way - to decode C structures encoded as byte strings). - .. availability:: not WASI. + .. method:: getsockopt(level, optname[, buflen]) + Return the value of the given socket option (see the Unix man page + :manpage:`getsockopt(2)`). The needed symbolic constants (:ref:`SO_\* etc. `) + are defined in this module. If *buflen* is absent, an integer option is assumed + and its integer value is returned by the function. If *buflen* is present, it + specifies the maximum length of the buffer used to receive the option in, and + this buffer is returned as a bytes object. It is up to the caller to decode the + contents of the buffer (see the optional built-in module :mod:`struct` for a way + to decode C structures encoded as byte strings). -.. method:: socket.getblocking() + .. availability:: not WASI. - Return ``True`` if socket is in blocking mode, ``False`` if in - non-blocking. - This is equivalent to checking ``socket.gettimeout() != 0``. + .. method:: getblocking() - .. versionadded:: 3.7 + Return ``True`` if socket is in blocking mode, ``False`` if in + non-blocking. + This is equivalent to checking ``socket.gettimeout() != 0``. -.. method:: socket.gettimeout() + .. versionadded:: 3.7 - Return the timeout in seconds (float) associated with socket operations, - or ``None`` if no timeout is set. This reflects the last call to - :meth:`setblocking` or :meth:`settimeout`. + .. method:: gettimeout() -.. method:: socket.ioctl(control, option) + Return the timeout in seconds (float) associated with socket operations, + or ``None`` if no timeout is set. This reflects the last call to + :meth:`setblocking` or :meth:`settimeout`. - The :meth:`ioctl` method is a limited interface to the WSAIoctl system - interface. Please refer to the `Win32 documentation - `_ for more - information. - On other platforms, the generic :func:`fcntl.fcntl` and :func:`fcntl.ioctl` - functions may be used; they accept a socket object as their first argument. + .. method:: ioctl(control, option) - Currently only the following control codes are supported: - ``SIO_RCVALL``, ``SIO_KEEPALIVE_VALS``, and ``SIO_LOOPBACK_FAST_PATH``. + The :meth:`ioctl` method is a limited interface to the WSAIoctl system + interface. Please refer to the `Win32 documentation + `_ for more + information. - .. availability:: Windows + On other platforms, the generic :func:`fcntl.fcntl` and :func:`fcntl.ioctl` + functions may be used; they accept a socket object as their first argument. - .. versionchanged:: 3.6 - ``SIO_LOOPBACK_FAST_PATH`` was added. + Currently only the following control codes are supported: + ``SIO_RCVALL``, ``SIO_KEEPALIVE_VALS``, and ``SIO_LOOPBACK_FAST_PATH``. + .. availability:: Windows -.. method:: socket.listen([backlog]) + .. versionchanged:: 3.6 + ``SIO_LOOPBACK_FAST_PATH`` was added. - Enable a server to accept connections. If *backlog* is specified, it must - be at least 0 (if it is lower, it is set to 0); it specifies the number of - unaccepted connections that the system will allow before refusing new - connections. If not specified, a default reasonable value is chosen. - .. availability:: not WASI. + .. method:: listen([backlog]) - .. versionchanged:: 3.5 - The *backlog* parameter is now optional. + Enable a server to accept connections. If *backlog* is specified, it must + be at least 0 (if it is lower, it is set to 0); it specifies the number of + unaccepted connections that the system will allow before refusing new + connections. If not specified, a default reasonable value is chosen. + .. availability:: not WASI. -.. method:: socket.makefile(mode='r', buffering=None, *, encoding=None, \ - errors=None, newline=None) + .. versionchanged:: 3.5 + The *backlog* parameter is now optional. - .. index:: single: I/O control; buffering - Return a :term:`file object` associated with the socket. The exact returned - type depends on the arguments given to :meth:`makefile`. These arguments are - interpreted the same way as by the built-in :func:`open` function, except - the only supported *mode* values are ``'r'`` (default), ``'w'``, ``'b'``, or - a combination of those. + .. method:: makefile(mode='r', buffering=None, *, encoding=None, \ + errors=None, newline=None) - The socket must be in blocking mode; it can have a timeout, but the file - object's internal buffer may end up in an inconsistent state if a timeout - occurs. + .. index:: single: I/O control; buffering - Closing the file object returned by :meth:`makefile` won't close the - original socket unless all other file objects have been closed and - :meth:`socket.close` has been called on the socket object. + Return a :term:`file object` associated with the socket. The exact returned + type depends on the arguments given to :meth:`makefile`. These arguments are + interpreted the same way as by the built-in :func:`open` function, except + the only supported *mode* values are ``'r'`` (default), ``'w'``, ``'b'``, or + a combination of those. - .. note:: + The socket must be in blocking mode; it can have a timeout, but the file + object's internal buffer may end up in an inconsistent state if a timeout + occurs. - On Windows, the file-like object created by :meth:`makefile` cannot be - used where a file object with a file descriptor is expected, such as the - stream arguments of :meth:`subprocess.Popen`. + Closing the file object returned by :meth:`makefile` won't close the + original socket unless all other file objects have been closed and + :meth:`socket.close` has been called on the socket object. + .. note:: -.. method:: socket.recv(bufsize[, flags]) + On Windows, the file-like object created by :meth:`makefile` cannot be + used where a file object with a file descriptor is expected, such as the + stream arguments of :meth:`subprocess.Popen`. - Receive data from the socket. The return value is a bytes object representing the - data received. The maximum amount of data to be received at once is specified - by *bufsize*. A returned empty bytes object indicates that the client has disconnected. - See the Unix manual page :manpage:`recv(2)` for the meaning of the optional argument - *flags*; it defaults to zero. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + .. method:: recv(bufsize[, flags]) + Receive data from the socket. The return value is a bytes object representing the + data received. The maximum amount of data to be received at once is specified + by *bufsize*. A returned empty bytes object indicates that the client has disconnected. + See the Unix manual page :manpage:`recv(2)` for the meaning of the optional argument + *flags*; it defaults to zero. -.. method:: socket.recvfrom(bufsize[, flags]) + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - Receive data from the socket. The return value is a pair ``(bytes, address)`` - where *bytes* is a bytes object representing the data received and *address* is the - address of the socket sending the data. See the Unix manual page - :manpage:`recv(2)` for the meaning of the optional argument *flags*; it defaults - to zero. (The format of *address* depends on the address family --- see above.) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + .. method:: recvfrom(bufsize[, flags]) - .. versionchanged:: 3.7 - For multicast IPv6 address, first item of *address* does not contain - ``%scope_id`` part anymore. In order to get full IPv6 address use - :func:`getnameinfo`. - -.. method:: socket.recvmsg(bufsize[, ancbufsize[, flags]]) - - Receive normal data (up to *bufsize* bytes) and ancillary data from - the socket. The *ancbufsize* argument sets the size in bytes of - the internal buffer used to receive the ancillary data; it defaults - to 0, meaning that no ancillary data will be received. Appropriate - buffer sizes for ancillary data can be calculated using - :func:`CMSG_SPACE` or :func:`CMSG_LEN`, and items which do not fit - into the buffer might be truncated or discarded. The *flags* - argument defaults to 0 and has the same meaning as for - :meth:`recv`. - - The return value is a 4-tuple: ``(data, ancdata, msg_flags, - address)``. The *data* item is a :class:`bytes` object holding the - non-ancillary data received. The *ancdata* item is a list of zero - or more tuples ``(cmsg_level, cmsg_type, cmsg_data)`` representing - the ancillary data (control messages) received: *cmsg_level* and - *cmsg_type* are integers specifying the protocol level and - protocol-specific type respectively, and *cmsg_data* is a - :class:`bytes` object holding the associated data. The *msg_flags* - item is the bitwise OR of various flags indicating conditions on - the received message; see your system documentation for details. - If the receiving socket is unconnected, *address* is the address of - the sending socket, if available; otherwise, its value is - unspecified. - - On some systems, :meth:`sendmsg` and :meth:`recvmsg` can be used to - pass file descriptors between processes over an :const:`AF_UNIX` - socket. When this facility is used (it is often restricted to - :const:`SOCK_STREAM` sockets), :meth:`recvmsg` will return, in its - ancillary data, items of the form ``(socket.SOL_SOCKET, - socket.SCM_RIGHTS, fds)``, where *fds* is a :class:`bytes` object - representing the new file descriptors as a binary array of the - native C :c:expr:`int` type. If :meth:`recvmsg` raises an - exception after the system call returns, it will first attempt to - close any file descriptors received via this mechanism. - - Some systems do not indicate the truncated length of ancillary data - items which have been only partially received. If an item appears - to extend beyond the end of the buffer, :meth:`recvmsg` will issue - a :exc:`RuntimeWarning`, and will return the part of it which is - inside the buffer provided it has not been truncated before the - start of its associated data. - - On systems which support the :const:`SCM_RIGHTS` mechanism, the - following function will receive up to *maxfds* file descriptors, - returning the message data and a list containing the descriptors - (while ignoring unexpected conditions such as unrelated control - messages being received). See also :meth:`sendmsg`. :: - - import socket, array - - def recv_fds(sock, msglen, maxfds): - fds = array.array("i") # Array of ints - msg, ancdata, flags, addr = sock.recvmsg(msglen, socket.CMSG_LEN(maxfds * fds.itemsize)) - for cmsg_level, cmsg_type, cmsg_data in ancdata: - if cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS: - # Append data, ignoring any truncated integers at the end. - fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) - return msg, list(fds) - - .. availability:: Unix. + Receive data from the socket. The return value is a pair ``(bytes, address)`` + where *bytes* is a bytes object representing the data received and *address* is the + address of the socket sending the data. See the Unix manual page + :manpage:`recv(2)` for the meaning of the optional argument *flags*; it defaults + to zero. The format of *address* depends on the address family --- see + :ref:`socket-addresses`. - Most Unix platforms. + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - .. versionadded:: 3.3 + .. versionchanged:: 3.7 + For multicast IPv6 address, first item of *address* does not contain + ``%scope_id`` part anymore. In order to get full IPv6 address use + :func:`getnameinfo`. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.recvmsg_into(buffers[, ancbufsize[, flags]]) - - Receive normal data and ancillary data from the socket, behaving as - :meth:`recvmsg` would, but scatter the non-ancillary data into a - series of buffers instead of returning a new bytes object. The - *buffers* argument must be an iterable of objects that export - writable buffers (e.g. :class:`bytearray` objects); these will be - filled with successive chunks of the non-ancillary data until it - has all been written or there are no more buffers. The operating - system may set a limit (:func:`~os.sysconf` value ``SC_IOV_MAX``) - on the number of buffers that can be used. The *ancbufsize* and - *flags* arguments have the same meaning as for :meth:`recvmsg`. - - The return value is a 4-tuple: ``(nbytes, ancdata, msg_flags, - address)``, where *nbytes* is the total number of bytes of - non-ancillary data written into the buffers, and *ancdata*, - *msg_flags* and *address* are the same as for :meth:`recvmsg`. - - Example:: - - >>> import socket - >>> s1, s2 = socket.socketpair() - >>> b1 = bytearray(b'----') - >>> b2 = bytearray(b'0123456789') - >>> b3 = bytearray(b'--------------') - >>> s1.send(b'Mary had a little lamb') - 22 - >>> s2.recvmsg_into([b1, memoryview(b2)[2:9], b3]) - (22, [], 0, None) - >>> [b1, b2, b3] - [bytearray(b'Mary'), bytearray(b'01 had a 9'), bytearray(b'little lamb---')] - - .. availability:: Unix. + .. method:: recvmsg(bufsize[, ancbufsize[, flags]]) + + Receive normal data (up to *bufsize* bytes) and ancillary data from + the socket. The *ancbufsize* argument sets the size in bytes of + the internal buffer used to receive the ancillary data; it defaults + to 0, meaning that no ancillary data will be received. Appropriate + buffer sizes for ancillary data can be calculated using + :func:`CMSG_SPACE` or :func:`CMSG_LEN`, and items which do not fit + into the buffer might be truncated or discarded. The *flags* + argument defaults to 0 and has the same meaning as for + :meth:`recv`. + + The return value is a 4-tuple: ``(data, ancdata, msg_flags, + address)``. The *data* item is a :class:`bytes` object holding the + non-ancillary data received. The *ancdata* item is a list of zero + or more tuples ``(cmsg_level, cmsg_type, cmsg_data)`` representing + the ancillary data (control messages) received: *cmsg_level* and + *cmsg_type* are integers specifying the protocol level and + protocol-specific type respectively, and *cmsg_data* is a + :class:`bytes` object holding the associated data. The *msg_flags* + item is the bitwise OR of various flags indicating conditions on + the received message; see your system documentation for details. + If the receiving socket is unconnected, *address* is the address of + the sending socket, if available; otherwise, its value is + unspecified. + + On some systems, :meth:`sendmsg` and :meth:`recvmsg` can be used to + pass file descriptors between processes over an :const:`AF_UNIX` + socket. When this facility is used (it is often restricted to + :const:`SOCK_STREAM` sockets), :meth:`recvmsg` will return, in its + ancillary data, items of the form ``(socket.SOL_SOCKET, + socket.SCM_RIGHTS, fds)``, where *fds* is a :class:`bytes` object + representing the new file descriptors as a binary array of the + native C :c:expr:`int` type. If :meth:`recvmsg` raises an + exception after the system call returns, it will first attempt to + close any file descriptors received via this mechanism. + + Some systems do not indicate the truncated length of ancillary data + items which have been only partially received. If an item appears + to extend beyond the end of the buffer, :meth:`recvmsg` will issue + a :exc:`RuntimeWarning`, and will return the part of it which is + inside the buffer provided it has not been truncated before the + start of its associated data. + + On systems which support the :const:`!SCM_RIGHTS` mechanism, the + following function will receive up to *maxfds* file descriptors, + returning the message data and a list containing the descriptors + (while ignoring unexpected conditions such as unrelated control + messages being received). See also :meth:`sendmsg`. :: + + import socket, array + + def recv_fds(sock, msglen, maxfds): + fds = array.array("i") # Array of ints + msg, ancdata, flags, addr = sock.recvmsg(msglen, socket.CMSG_LEN(maxfds * fds.itemsize)) + for cmsg_level, cmsg_type, cmsg_data in ancdata: + if cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS: + # Append data, ignoring any truncated integers at the end. + fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + return msg, list(fds) + + .. availability:: Unix. + + Most Unix platforms. + + .. versionadded:: 3.3 + + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + + + .. method:: recvmsg_into(buffers[, ancbufsize[, flags]]) + + Receive normal data and ancillary data from the socket, behaving as + :meth:`recvmsg` would, but scatter the non-ancillary data into a + series of buffers instead of returning a new bytes object. The + *buffers* argument must be an iterable of objects that export + writable buffers (e.g. :class:`bytearray` objects); these will be + filled with successive chunks of the non-ancillary data until it + has all been written or there are no more buffers. The operating + system may set a limit (:func:`~os.sysconf` value ``SC_IOV_MAX``) + on the number of buffers that can be used. The *ancbufsize* and + *flags* arguments have the same meaning as for :meth:`recvmsg`. + + The return value is a 4-tuple: ``(nbytes, ancdata, msg_flags, + address)``, where *nbytes* is the total number of bytes of + non-ancillary data written into the buffers, and *ancdata*, + *msg_flags* and *address* are the same as for :meth:`recvmsg`. + + Example:: + + >>> import socket + >>> s1, s2 = socket.socketpair() + >>> b1 = bytearray(b'----') + >>> b2 = bytearray(b'0123456789') + >>> b3 = bytearray(b'--------------') + >>> s1.send(b'Mary had a little lamb') + 22 + >>> s2.recvmsg_into([b1, memoryview(b2)[2:9], b3]) + (22, [], 0, None) + >>> [b1, b2, b3] + [bytearray(b'Mary'), bytearray(b'01 had a 9'), bytearray(b'little lamb---')] + + .. availability:: Unix. + + Most Unix platforms. + + .. versionadded:: 3.3 + + + .. method:: recvfrom_into(buffer[, nbytes[, flags]]) + + Receive data from the socket, writing it into *buffer* instead of creating a + new bytestring. The return value is a pair ``(nbytes, address)`` where *nbytes* is + the number of bytes received and *address* is the address of the socket sending + the data. See the Unix manual page :manpage:`recv(2)` for the meaning of the + optional argument *flags*; it defaults to zero. The format of *address* + depends on the address family --- see :ref:`socket-addresses`. + + + .. method:: recv_into(buffer[, nbytes[, flags]]) + + Receive up to *nbytes* bytes from the socket, storing the data into a buffer + rather than creating a new bytestring. If *nbytes* is not specified (or 0), + receive up to the size available in the given buffer. Returns the number of + bytes received. See the Unix manual page :manpage:`recv(2)` for the meaning + of the optional argument *flags*; it defaults to zero. - Most Unix platforms. - .. versionadded:: 3.3 + .. method:: send(bytes[, flags]) + Send data to the socket. The socket must be connected to a remote socket. The + optional *flags* argument has the same meaning as for :meth:`recv`. + Returns the number of bytes sent. Applications are responsible for checking that + all data has been sent; if only some of the data was transmitted, the + application needs to attempt delivery of the remaining data. For further + information on this topic, consult the :ref:`socket-howto`. -.. method:: socket.recvfrom_into(buffer[, nbytes[, flags]]) + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - Receive data from the socket, writing it into *buffer* instead of creating a - new bytestring. The return value is a pair ``(nbytes, address)`` where *nbytes* is - the number of bytes received and *address* is the address of the socket sending - the data. See the Unix manual page :manpage:`recv(2)` for the meaning of the - optional argument *flags*; it defaults to zero. (The format of *address* - depends on the address family --- see above.) + .. method:: sendall(bytes[, flags]) -.. method:: socket.recv_into(buffer[, nbytes[, flags]]) + Send data to the socket. The socket must be connected to a remote socket. The + optional *flags* argument has the same meaning as for :meth:`recv`. + Unlike :meth:`send`, this method continues to send data from *bytes* until + either all data has been sent or an error occurs. ``None`` is returned on + success. On error, an exception is raised, and there is no way to determine how + much data, if any, was successfully sent. - Receive up to *nbytes* bytes from the socket, storing the data into a buffer - rather than creating a new bytestring. If *nbytes* is not specified (or 0), - receive up to the size available in the given buffer. Returns the number of - bytes received. See the Unix manual page :manpage:`recv(2)` for the meaning - of the optional argument *flags*; it defaults to zero. + .. versionchanged:: 3.5 + The socket timeout is no longer reset each time data is sent successfully. + The socket timeout is now the maximum total duration to send all data. + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). -.. method:: socket.send(bytes[, flags]) - Send data to the socket. The socket must be connected to a remote socket. The - optional *flags* argument has the same meaning as for :meth:`recv` above. - Returns the number of bytes sent. Applications are responsible for checking that - all data has been sent; if only some of the data was transmitted, the - application needs to attempt delivery of the remaining data. For further - information on this topic, consult the :ref:`socket-howto`. + .. method:: sendto(bytes, address) + sendto(bytes, flags, address) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + Send data to the socket. The socket should not be connected to a remote socket, + since the destination socket is specified by *address*. The optional *flags* + argument has the same meaning as for :meth:`recv`. Return the number of + bytes sent. The format of *address* depends on the address family --- see + :ref:`socket-addresses`. + .. audit-event:: socket.sendto self,address socket.socket.sendto -.. method:: socket.sendall(bytes[, flags]) + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - Send data to the socket. The socket must be connected to a remote socket. The - optional *flags* argument has the same meaning as for :meth:`recv` above. - Unlike :meth:`send`, this method continues to send data from *bytes* until - either all data has been sent or an error occurs. ``None`` is returned on - success. On error, an exception is raised, and there is no way to determine how - much data, if any, was successfully sent. - .. versionchanged:: 3.5 - The socket timeout is no longer reset each time data is sent successfully. - The socket timeout is now the maximum total duration to send all data. + .. method:: sendmsg(buffers[, ancdata[, flags[, address]]]) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + Send normal and ancillary data to the socket, gathering the + non-ancillary data from a series of buffers and concatenating it + into a single message. The *buffers* argument specifies the + non-ancillary data as an iterable of + :term:`bytes-like objects ` + (e.g. :class:`bytes` objects); the operating system may set a limit + (:func:`~os.sysconf` value ``SC_IOV_MAX``) on the number of buffers + that can be used. The *ancdata* argument specifies the ancillary + data (control messages) as an iterable of zero or more tuples + ``(cmsg_level, cmsg_type, cmsg_data)``, where *cmsg_level* and + *cmsg_type* are integers specifying the protocol level and + protocol-specific type respectively, and *cmsg_data* is a + bytes-like object holding the associated data. Note that + some systems (in particular, systems without :func:`CMSG_SPACE`) + might support sending only one control message per call. The + *flags* argument defaults to 0 and has the same meaning as for + :meth:`send`. If *address* is supplied and not ``None``, it sets a + destination address for the message. The return value is the + number of bytes of non-ancillary data sent. + The following function sends the list of file descriptors *fds* + over an :const:`AF_UNIX` socket, on systems which support the + :const:`!SCM_RIGHTS` mechanism. See also :meth:`recvmsg`. :: -.. method:: socket.sendto(bytes, address) - socket.sendto(bytes, flags, address) + import socket, array - Send data to the socket. The socket should not be connected to a remote socket, - since the destination socket is specified by *address*. The optional *flags* - argument has the same meaning as for :meth:`recv` above. Return the number of - bytes sent. (The format of *address* depends on the address family --- see - above.) + def send_fds(sock, msg, fds): + return sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", fds))]) - .. audit-event:: socket.sendto self,address socket.socket.sendto + .. availability:: Unix, not WASI. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.sendmsg(buffers[, ancdata[, flags[, address]]]) - - Send normal and ancillary data to the socket, gathering the - non-ancillary data from a series of buffers and concatenating it - into a single message. The *buffers* argument specifies the - non-ancillary data as an iterable of - :term:`bytes-like objects ` - (e.g. :class:`bytes` objects); the operating system may set a limit - (:func:`~os.sysconf` value ``SC_IOV_MAX``) on the number of buffers - that can be used. The *ancdata* argument specifies the ancillary - data (control messages) as an iterable of zero or more tuples - ``(cmsg_level, cmsg_type, cmsg_data)``, where *cmsg_level* and - *cmsg_type* are integers specifying the protocol level and - protocol-specific type respectively, and *cmsg_data* is a - bytes-like object holding the associated data. Note that - some systems (in particular, systems without :func:`CMSG_SPACE`) - might support sending only one control message per call. The - *flags* argument defaults to 0 and has the same meaning as for - :meth:`send`. If *address* is supplied and not ``None``, it sets a - destination address for the message. The return value is the - number of bytes of non-ancillary data sent. - - The following function sends the list of file descriptors *fds* - over an :const:`AF_UNIX` socket, on systems which support the - :const:`SCM_RIGHTS` mechanism. See also :meth:`recvmsg`. :: - - import socket, array - - def send_fds(sock, msg, fds): - return sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", fds))]) + Most Unix platforms. - .. availability:: Unix, not WASI. + .. audit-event:: socket.sendmsg self,address socket.socket.sendmsg - Most Unix platforms. + .. versionadded:: 3.3 - .. audit-event:: socket.sendmsg self,address socket.socket.sendmsg + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - .. versionadded:: 3.3 + .. method:: sendmsg_afalg([msg], *, op[, iv[, assoclen[, flags]]]) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + Specialized version of :meth:`~socket.sendmsg` for :const:`AF_ALG` socket. + Set mode, IV, AEAD associated data length and flags for :const:`AF_ALG` socket. -.. method:: socket.sendmsg_afalg([msg], *, op[, iv[, assoclen[, flags]]]) + .. availability:: Linux >= 2.6.38. - Specialized version of :meth:`~socket.sendmsg` for :const:`AF_ALG` socket. - Set mode, IV, AEAD associated data length and flags for :const:`AF_ALG` socket. + .. versionadded:: 3.6 - .. availability:: Linux >= 2.6.38. + .. method:: sendfile(file, offset=0, count=None) - .. versionadded:: 3.6 + Send a file until EOF is reached by using high-performance + :mod:`os.sendfile` and return the total number of bytes which were sent. + *file* must be a regular file object opened in binary mode. If + :mod:`os.sendfile` is not available (e.g. Windows) or *file* is not a + regular file :meth:`send` will be used instead. *offset* tells from where to + start reading the file. If specified, *count* is the total number of bytes + to transmit as opposed to sending the file until EOF is reached. File + position is updated on return or also in case of error in which case + :meth:`file.tell() ` can be used to figure out the number of + bytes which were sent. The socket must be of :const:`SOCK_STREAM` type. + Non-blocking sockets are not supported. -.. method:: socket.sendfile(file, offset=0, count=None) + .. versionadded:: 3.5 - Send a file until EOF is reached by using high-performance - :mod:`os.sendfile` and return the total number of bytes which were sent. - *file* must be a regular file object opened in binary mode. If - :mod:`os.sendfile` is not available (e.g. Windows) or *file* is not a - regular file :meth:`send` will be used instead. *offset* tells from where to - start reading the file. If specified, *count* is the total number of bytes - to transmit as opposed to sending the file until EOF is reached. File - position is updated on return or also in case of error in which case - :meth:`file.tell() ` can be used to figure out the number of - bytes which were sent. The socket must be of :const:`SOCK_STREAM` type. - Non-blocking sockets are not supported. + .. method:: set_inheritable(inheritable) - .. versionadded:: 3.5 + Set the :ref:`inheritable flag ` of the socket's file + descriptor or socket's handle. -.. method:: socket.set_inheritable(inheritable) + .. versionadded:: 3.4 - Set the :ref:`inheritable flag ` of the socket's file - descriptor or socket's handle. - .. versionadded:: 3.4 + .. method:: setblocking(flag) + Set blocking or non-blocking mode of the socket: if *flag* is false, the + socket is set to non-blocking, else to blocking mode. -.. method:: socket.setblocking(flag) + This method is a shorthand for certain :meth:`~socket.settimeout` calls: - Set blocking or non-blocking mode of the socket: if *flag* is false, the - socket is set to non-blocking, else to blocking mode. + * ``sock.setblocking(True)`` is equivalent to ``sock.settimeout(None)`` - This method is a shorthand for certain :meth:`~socket.settimeout` calls: + * ``sock.setblocking(False)`` is equivalent to ``sock.settimeout(0.0)`` - * ``sock.setblocking(True)`` is equivalent to ``sock.settimeout(None)`` + .. versionchanged:: 3.7 + The method no longer applies :const:`SOCK_NONBLOCK` flag on + :attr:`socket.type`. - * ``sock.setblocking(False)`` is equivalent to ``sock.settimeout(0.0)`` - .. versionchanged:: 3.7 - The method no longer applies :const:`SOCK_NONBLOCK` flag on - :attr:`socket.type`. + .. method:: settimeout(value) + Set a timeout on blocking socket operations. The *value* argument can be a + nonnegative real number expressing seconds, or ``None``. + If a non-zero value is given, subsequent socket operations will raise a + :exc:`timeout` exception if the timeout period *value* has elapsed before + the operation has completed. If zero is given, the socket is put in + non-blocking mode. If ``None`` is given, the socket is put in blocking mode. -.. method:: socket.settimeout(value) + For further information, please consult the :ref:`notes on socket timeouts `. - Set a timeout on blocking socket operations. The *value* argument can be a - nonnegative floating-point number expressing seconds, or ``None``. - If a non-zero value is given, subsequent socket operations will raise a - :exc:`timeout` exception if the timeout period *value* has elapsed before - the operation has completed. If zero is given, the socket is put in - non-blocking mode. If ``None`` is given, the socket is put in blocking mode. + .. versionchanged:: 3.7 + The method no longer toggles :const:`SOCK_NONBLOCK` flag on + :attr:`socket.type`. - For further information, please consult the :ref:`notes on socket timeouts `. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. - .. versionchanged:: 3.7 - The method no longer toggles :const:`SOCK_NONBLOCK` flag on - :attr:`socket.type`. + .. method:: setsockopt(level, optname, value: int | Buffer) + setsockopt(level, optname, None, optlen: int) -.. method:: socket.setsockopt(level, optname, value: int) -.. method:: socket.setsockopt(level, optname, value: buffer) - :noindex: -.. method:: socket.setsockopt(level, optname, None, optlen: int) - :noindex: + .. index:: pair: module; struct - .. index:: pair: module; struct + Set the value of the given socket option (see the Unix manual page + :manpage:`setsockopt(2)`). The needed symbolic constants are defined in this + module (:ref:`!SO_\* etc. `). The value can be an integer, + ``None`` or a :term:`bytes-like object` representing a buffer. In the latter + case it is up to the caller to ensure that the bytestring contains the + proper bits (see the optional built-in module :mod:`struct` for a way to + encode C structures as bytestrings). When *value* is set to ``None``, + *optlen* argument is required. It's equivalent to calling :c:func:`!setsockopt` C + function with ``optval=NULL`` and ``optlen=optlen``. - Set the value of the given socket option (see the Unix manual page - :manpage:`setsockopt(2)`). The needed symbolic constants are defined in this - module (:ref:`!SO_\* etc. `). The value can be an integer, - ``None`` or a :term:`bytes-like object` representing a buffer. In the later - case it is up to the caller to ensure that the bytestring contains the - proper bits (see the optional built-in module :mod:`struct` for a way to - encode C structures as bytestrings). When *value* is set to ``None``, - *optlen* argument is required. It's equivalent to call :c:func:`setsockopt` C - function with ``optval=NULL`` and ``optlen=optlen``. + .. versionchanged:: 3.5 + Writable :term:`bytes-like object` is now accepted. - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. + .. versionchanged:: 3.6 + setsockopt(level, optname, None, optlen: int) form added. - .. versionchanged:: 3.6 - setsockopt(level, optname, None, optlen: int) form added. + .. availability:: not WASI. - .. availability:: not WASI. + .. method:: shutdown(how) -.. method:: socket.shutdown(how) + Shut down one or both halves of the connection. If *how* is :const:`SHUT_RD`, + further receives are disallowed. If *how* is :const:`SHUT_WR`, further sends + are disallowed. If *how* is :const:`SHUT_RDWR`, further sends and receives are + disallowed. - Shut down one or both halves of the connection. If *how* is :const:`SHUT_RD`, - further receives are disallowed. If *how* is :const:`SHUT_WR`, further sends - are disallowed. If *how* is :const:`SHUT_RDWR`, further sends and receives are - disallowed. + .. availability:: not WASI. - .. availability:: not WASI. + .. method:: share(process_id) -.. method:: socket.share(process_id) + Duplicate a socket and prepare it for sharing with a target process. The + target process must be provided with *process_id*. The resulting bytes object + can then be passed to the target process using some form of interprocess + communication and the socket can be recreated there using :func:`fromshare`. + Once this method has been called, it is safe to close the socket since + the operating system has already duplicated it for the target process. - Duplicate a socket and prepare it for sharing with a target process. The - target process must be provided with *process_id*. The resulting bytes object - can then be passed to the target process using some form of interprocess - communication and the socket can be recreated there using :func:`fromshare`. - Once this method has been called, it is safe to close the socket since - the operating system has already duplicated it for the target process. + .. availability:: Windows. - .. availability:: Windows. + .. versionadded:: 3.3 - .. versionadded:: 3.3 + Note that there are no methods :meth:`!read` or :meth:`!write`; use + :meth:`~socket.recv` and :meth:`~socket.send` without *flags* argument instead. + + Socket objects also have these (read-only) attributes that correspond to the + values given to the :class:`~socket.socket` constructor. -Note that there are no methods :meth:`read` or :meth:`write`; use -:meth:`~socket.recv` and :meth:`~socket.send` without *flags* argument instead. -Socket objects also have these (read-only) attributes that correspond to the -values given to the :class:`~socket.socket` constructor. + .. attribute:: family + The socket family. -.. attribute:: socket.family - The socket family. + .. attribute:: type + The socket type. -.. attribute:: socket.type - The socket type. + .. attribute:: proto + The socket protocol. -.. attribute:: socket.proto - The socket protocol. +.. class:: SocketType + The base class of the :class:`~socket.socket` type, re-exported from + :mod:`!_socket`. An instance check such as + ``isinstance(socket(...), SocketType)`` is true, but ``SocketType`` is not + the same as ``type(socket(...))``, which is :class:`~socket.socket` itself. .. _socket-timeouts: @@ -2415,7 +2435,7 @@ lead to this error:: This is because the previous execution has left the socket in a ``TIME_WAIT`` state, and can't be immediately reused. -There is a :mod:`socket` flag to set, in order to prevent this, +There is a :mod:`!socket` flag to set, in order to prevent this, :const:`socket.SO_REUSEADDR`:: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 7fb629f7d2f256c..4c98bb8e3b9c9bf 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`socketserver` module simplifies the task of writing network servers. +The :mod:`!socketserver` module simplifies the task of writing network servers. .. include:: ../includes/wasm-notavail.rst @@ -24,7 +24,7 @@ There are four basic concrete server classes: :meth:`~BaseServer.server_activate`. The other parameters are passed to the :class:`BaseServer` base class. - .. versionchanged:: next + .. versionchanged:: 3.15 The default queue size is now ``socket.SOMAXCONN`` for :class:`socketserver.TCPServer`. .. class:: UDPServer(server_address, RequestHandlerClass, bind_and_activate=True) @@ -546,7 +546,7 @@ The difference is that the ``readline()`` call in the second handler will call first handler had to use a ``recv()`` loop to accumulate data until a newline itself. If it had just used a single ``recv()`` without the loop it would just have returned what has been received so far from the client. -TCP is stream based: data arrives in the order it was sent, but there no +TCP is stream based: data arrives in the order it was sent, but there is no correlation between client ``send()`` or ``sendall()`` calls and the number of ``recv()`` calls on the server required to receive it. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index a14af6d3d88df27..3a75d44f3f7d21b 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -4,8 +4,6 @@ .. module:: sqlite3 :synopsis: A DB-API 2.0 implementation using SQLite 3.x. -.. sectionauthor:: Gerhard Häring - **Source code:** :source:`Lib/sqlite3/` .. Make sure we always doctest the tutorial with an empty database. @@ -31,7 +29,9 @@ PostgreSQL or Oracle. The :mod:`!sqlite3` module was written by Gerhard Häring. It provides an SQL interface compliant with the DB-API 2.0 specification described by :pep:`249`, and -requires SQLite 3.15.2 or newer. +requires the third-party `SQLite `_ library. + +.. include:: ../includes/optional-module.rst This document includes four main sections: @@ -55,7 +55,7 @@ This document includes four main sections: PEP written by Marc-André Lemburg. -.. We use the following practises for SQL code: +.. We use the following practices for SQL code: - UPPERCASE for keywords - snake_case for schema - single quotes for string literals @@ -289,7 +289,7 @@ Module functions Set it to any combination (using ``|``, bitwise or) of :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES` to enable this. - Column names takes precedence over declared types if both flags are set. + Column names take precedence over declared types if both flags are set. By default (``0``), type detection is disabled. :param isolation_level: @@ -509,12 +509,12 @@ Module constants .. data:: SQLITE_KEYWORDS - A :class:`tuple` containing all sqlite3 keywords. + A :class:`tuple` containing all SQLite keywords. This constant is only available if Python was compiled with SQLite 3.24.0 or greater. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: threadsafety @@ -620,7 +620,7 @@ Connection objects supplied, this must be a :term:`callable` returning an instance of :class:`Cursor` or its subclasses. - .. method:: blobopen(table, column, row, /, *, readonly=False, name="main") + .. method:: blobopen(table, column, rowid, /, *, readonly=False, name="main") Open a :class:`Blob` handle to an existing :abbr:`BLOB (Binary Large OBject)`. @@ -631,8 +631,8 @@ Connection objects :param str column: The name of the column where the blob is located. - :param str row: - The name of the row where the blob is located. + :param int rowid: + The row id where the blob is located. :param bool readonly: Set to ``True`` if the blob should be opened without write @@ -1417,6 +1417,9 @@ Connection objects See :ref:`sqlite3-howto-row-factory` for more details. + .. versionchanged:: next + Deleting the ``row_factory`` attribute is no longer allowed. + .. attribute:: text_factory A :term:`callable` that accepts a :class:`bytes` parameter @@ -1426,6 +1429,9 @@ Connection objects See :ref:`sqlite3-howto-encoding` for more details. + .. versionchanged:: next + Deleting the ``text_factory`` attribute is no longer allowed. + .. attribute:: total_changes Return the total number of database rows that have been modified, inserted, or @@ -1611,6 +1617,9 @@ Cursor objects If the *size* parameter is used, then it is best for it to retain the same value from one :meth:`fetchmany` call to the next. + .. versionchanged:: 3.15 + Negative *size* values are rejected by raising :exc:`ValueError`. + .. method:: fetchall() Return all (remaining) rows of a query result as a :class:`list`. @@ -1638,6 +1647,9 @@ Cursor objects Read/write attribute that controls the number of rows returned by :meth:`fetchmany`. The default value is 1 which means a single row would be fetched per call. + .. versionchanged:: 3.15 + Negative values are rejected by raising :exc:`ValueError`. + .. attribute:: connection Read-only attribute that provides the SQLite database :class:`Connection` @@ -1703,6 +1715,9 @@ Cursor objects See :ref:`sqlite3-howto-row-factory` for more details. + .. versionchanged:: next + Deleting the ``row_factory`` attribute is no longer allowed. + .. The sqlite3.Row example used to be a how-to. It has now been incorporated into the Row reference. We keep the anchor here in order not to break @@ -2279,7 +2294,7 @@ This section shows recipes for common adapters and converters. .. testcode:: - import datetime + import datetime as dt import sqlite3 def adapt_date_iso(val): @@ -2294,21 +2309,21 @@ This section shows recipes for common adapters and converters. """Adapt datetime.datetime to Unix timestamp.""" return int(val.timestamp()) - sqlite3.register_adapter(datetime.date, adapt_date_iso) - sqlite3.register_adapter(datetime.datetime, adapt_datetime_iso) - sqlite3.register_adapter(datetime.datetime, adapt_datetime_epoch) + sqlite3.register_adapter(dt.date, adapt_date_iso) + sqlite3.register_adapter(dt.datetime, adapt_datetime_iso) + sqlite3.register_adapter(dt.datetime, adapt_datetime_epoch) def convert_date(val): """Convert ISO 8601 date to datetime.date object.""" - return datetime.date.fromisoformat(val.decode()) + return dt.date.fromisoformat(val.decode()) def convert_datetime(val): """Convert ISO 8601 datetime to datetime.datetime object.""" - return datetime.datetime.fromisoformat(val.decode()) + return dt.datetime.fromisoformat(val.decode()) def convert_timestamp(val): """Convert Unix epoch timestamp to datetime.datetime object.""" - return datetime.datetime.fromtimestamp(int(val)) + return dt.datetime.fromtimestamp(int(val)) sqlite3.register_converter("date", convert_date) sqlite3.register_converter("datetime", convert_datetime) @@ -2317,17 +2332,17 @@ This section shows recipes for common adapters and converters. .. testcode:: :hide: - dt = datetime.datetime(2019, 5, 18, 15, 17, 8, 123456) + when = dt.datetime(2019, 5, 18, 15, 17, 8, 123456) - assert adapt_date_iso(dt.date()) == "2019-05-18" - assert convert_date(b"2019-05-18") == dt.date() + assert adapt_date_iso(when.date()) == "2019-05-18" + assert convert_date(b"2019-05-18") == when.date() - assert adapt_datetime_iso(dt) == "2019-05-18T15:17:08.123456" - assert convert_datetime(b"2019-05-18T15:17:08.123456") == dt + assert adapt_datetime_iso(when) == "2019-05-18T15:17:08.123456" + assert convert_datetime(b"2019-05-18T15:17:08.123456") == when # Using current time as fromtimestamp() returns local date/time. # Dropping microseconds as adapt_datetime_epoch truncates fractional second part. - now = datetime.datetime.now().replace(microsecond=0) + now = dt.datetime.now().replace(microsecond=0) current_timestamp = int(now.timestamp()) assert adapt_datetime_epoch(now) == current_timestamp diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index a9930183f9a4006..66fe6c7aee48626 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -4,9 +4,6 @@ .. module:: ssl :synopsis: TLS/SSL wrapper for socket objects -.. moduleauthor:: Bill Janssen -.. sectionauthor:: Bill Janssen - **Source code:** :source:`Lib/ssl.py` .. index:: single: OpenSSL; (use in module ssl) @@ -18,8 +15,9 @@ This module provides access to Transport Layer Security (often known as "Secure Sockets Layer") encryption and peer authentication facilities for network sockets, both client-side and server-side. This module uses the OpenSSL -library. It is available on all modern Unix systems, Windows, macOS, and -probably additional platforms, as long as OpenSSL is installed on that platform. +library. + +.. include:: ../includes/optional-module.rst .. note:: @@ -69,7 +67,7 @@ by SSL sockets created through the :meth:`SSLContext.wrap_socket` method. Use of deprecated constants and functions result in deprecation warnings. -Functions, Constants, and Exceptions +Functions, constants, and exceptions ------------------------------------ @@ -125,10 +123,11 @@ Context creation A convenience function helps create :class:`SSLContext` objects for common purposes. -.. function:: create_default_context(purpose=Purpose.SERVER_AUTH, cafile=None, capath=None, cadata=None) +.. function:: create_default_context(purpose=Purpose.SERVER_AUTH, *,\ + cafile=None, capath=None, cadata=None) Return a new :class:`SSLContext` object with default settings for - the given *purpose*. The settings are chosen by the :mod:`ssl` module, + the given *purpose*. The settings are chosen by the :mod:`!ssl` module, and usually represent a higher security level than when calling the :class:`SSLContext` constructor directly. @@ -147,9 +146,9 @@ purposes. *cadata* is given) or uses :meth:`SSLContext.load_default_certs` to load default CA certificates. - When :attr:`~SSLContext.keylog_filename` is supported and the environment - variable :envvar:`SSLKEYLOGFILE` is set, :func:`create_default_context` - enables key logging. + When the environment variable :envvar:`!SSLKEYLOGFILE` is set, + :func:`create_default_context` enables key logging by setting + :attr:`~SSLContext.keylog_filename` to the variable's value. The default settings for this context include :data:`VERIFY_X509_PARTIAL_CHAIN` and :data:`VERIFY_X509_STRICT`. @@ -215,6 +214,25 @@ purposes. :data:`VERIFY_X509_STRICT` in its default verify flags. +Signature algorithms +^^^^^^^^^^^^^^^^^^^^ + +.. function:: get_sigalgs() + + Return a list of available TLS signature algorithm names used + by servers to complete the TLS handshake or clients requesting + certificate-based authentication. For example:: + + >>> ssl.get_sigalgs() # doctest: +SKIP + ['ecdsa_secp256r1_sha256', 'ecdsa_secp384r1_sha384', ...] + + These names can be used when building string values to pass to the + :meth:`SSLContext.set_client_sigalgs` and + :meth:`SSLContext.set_server_sigalgs` methods. + + .. versionadded:: 3.15 + + Exceptions ^^^^^^^^^^ @@ -314,7 +332,7 @@ Exceptions Random generation ^^^^^^^^^^^^^^^^^ -.. function:: RAND_bytes(num) +.. function:: RAND_bytes(num, /) Return *num* cryptographically strong pseudo-random bytes. Raises an :class:`SSLError` if the PRNG has not been seeded with enough data or if the @@ -334,11 +352,10 @@ Random generation .. function:: RAND_status() Return ``True`` if the SSL pseudo-random number generator has been seeded - with 'enough' randomness, and ``False`` otherwise. You can use - :func:`ssl.RAND_egd` and :func:`ssl.RAND_add` to increase the randomness of - the pseudo-random number generator. + with 'enough' randomness, and ``False`` otherwise. Use :func:`ssl.RAND_add` + to increase the randomness of the pseudo-random number generator. -.. function:: RAND_add(bytes, entropy) +.. function:: RAND_add(bytes, entropy, /) Mix the given *bytes* into the SSL pseudo-random number generator. The parameter *entropy* (a float) is a lower bound on the entropy contained in @@ -357,7 +374,7 @@ Certificate handling .. function:: cert_time_to_seconds(cert_time) - Return the time in seconds since the Epoch, given the ``cert_time`` + Return the time in seconds since the epoch, given the ``cert_time`` string representing the "notBefore" or "notAfter" date from a certificate in ``"%b %d %H:%M:%S %Y %Z"`` strptime format (C locale). @@ -367,12 +384,12 @@ Certificate handling .. doctest:: newcontext >>> import ssl + >>> import datetime as dt >>> timestamp = ssl.cert_time_to_seconds("Jan 5 09:34:43 2018 GMT") >>> timestamp # doctest: +SKIP 1515144883 - >>> from datetime import datetime - >>> print(datetime.utcfromtimestamp(timestamp)) # doctest: +SKIP - 2018-01-05 09:34:43 + >>> print(dt.datetime.fromtimestamp(timestamp, dt.UTC)) # doctest: +SKIP + 2018-01-05 09:34:43+00:00 "notBefore" or "notAfter" dates must use GMT (:rfc:`5280`). @@ -406,12 +423,12 @@ Certificate handling .. versionchanged:: 3.10 The *timeout* parameter was added. -.. function:: DER_cert_to_PEM_cert(DER_cert_bytes) +.. function:: DER_cert_to_PEM_cert(der_cert_bytes) Given a certificate as a DER-encoded blob of bytes, returns a PEM-encoded string version of the same certificate. -.. function:: PEM_cert_to_DER_cert(PEM_cert_string) +.. function:: PEM_cert_to_DER_cert(pem_cert_string) Given a certificate as an ASCII PEM string, returns a DER-encoded sequence of bytes for that same certificate. @@ -939,7 +956,7 @@ Constants Whether the OpenSSL library has built-in support for External PSKs in TLS 1.3 as described in :rfc:`9258`. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: HAS_PHA @@ -1055,7 +1072,7 @@ Constants :attr:`TLSVersion.TLSv1_3` are deprecated. -SSL Sockets +SSL sockets ----------- .. class:: SSLSocket(socket.socket) @@ -1104,7 +1121,7 @@ SSL Sockets :meth:`SSLContext.wrap_socket` to wrap a socket. .. versionchanged:: 3.7 - :class:`SSLSocket` instances must to created with + :class:`SSLSocket` instances must be created with :meth:`~SSLContext.wrap_socket`. In earlier versions, it was possible to create instances directly. This was never documented or officially supported. @@ -1114,7 +1131,7 @@ SSL Sockets functions support reading and writing of data larger than 2 GB. Writing zero-length data no longer fails with a protocol violation error. - .. versionchanged:: next + .. versionchanged:: 3.15 Python now uses ``SSL_sendfile`` internally when possible. The function sends a file more efficiently because it performs TLS encryption in the kernel to avoid additional context switches. @@ -1141,10 +1158,10 @@ SSL sockets also have the following additional methods and attributes: .. deprecated:: 3.6 Use :meth:`~SSLSocket.recv` instead of :meth:`~SSLSocket.read`. -.. method:: SSLSocket.write(buf) +.. method:: SSLSocket.write(data) - Write *buf* to the SSL socket and return the number of bytes written. The - *buf* argument must be an object supporting the buffer interface. + Write *data* to the SSL socket and return the number of bytes written. The + *data* argument must be an object supporting the buffer interface. Raise :exc:`SSLWantReadError` or :exc:`SSLWantWriteError` if the socket is :ref:`non-blocking ` and the write would block. @@ -1154,7 +1171,7 @@ SSL sockets also have the following additional methods and attributes: .. versionchanged:: 3.5 The socket timeout is no longer reset each time bytes are received or sent. - The socket timeout is now the maximum total duration to write *buf*. + The socket timeout is now the maximum total duration to write *data*. .. deprecated:: 3.6 Use :meth:`~SSLSocket.send` instead of :meth:`~SSLSocket.write`. @@ -1171,12 +1188,15 @@ SSL sockets also have the following additional methods and attributes: :meth:`~socket.socket.recv` and :meth:`~socket.socket.send` instead of these methods. -.. method:: SSLSocket.do_handshake() +.. method:: SSLSocket.do_handshake(block=False) Perform the SSL setup handshake. + If *block* is true and the timeout obtained by :meth:`~socket.socket.gettimeout` + is zero, the socket is set in blocking mode until the handshake is performed. + .. versionchanged:: 3.4 - The handshake method also performs :func:`match_hostname` when the + The handshake method also performs :func:`!match_hostname` when the :attr:`~SSLContext.check_hostname` attribute of the socket's :attr:`~SSLSocket.context` is true. @@ -1186,7 +1206,7 @@ SSL sockets also have the following additional methods and attributes: .. versionchanged:: 3.7 Hostname or IP address is matched by OpenSSL during handshake. The - function :func:`match_hostname` is no longer used. In case OpenSSL + function :func:`!match_hostname` is no longer used. In case OpenSSL refuses a hostname or IP address, the handshake is aborted early and a TLS alert message is sent to the peer. @@ -1290,6 +1310,29 @@ SSL sockets also have the following additional methods and attributes: .. versionadded:: 3.5 +.. method:: SSLSocket.group() + + Return the group used for doing key agreement on this connection. If no + connection has been established, returns ``None``. + + .. versionadded:: 3.15 + +.. method:: SSLSocket.client_sigalg() + + Return the signature algorithm used for performing certificate-based client + authentication on this connection, or ``None`` if no connection has been + established or client authentication didn't occur. + + .. versionadded:: 3.15 + +.. method:: SSLSocket.server_sigalg() + + Return the signature algorithm used by the server to complete the TLS + handshake on this connection, or ``None`` if no connection has been + established or the cipher suite has no signature. + + .. versionadded:: 3.15 + .. method:: SSLSocket.compression() Return the compression algorithm being used as a string, or ``None`` @@ -1419,7 +1462,7 @@ SSL sockets also have the following additional methods and attributes: .. versionadded:: 3.6 -SSL Contexts +SSL contexts ------------ .. versionadded:: 3.2 @@ -1464,7 +1507,7 @@ to speed up repeated connections from the same clients. TLS 1.3. .. seealso:: - :func:`create_default_context` lets the :mod:`ssl` module choose + :func:`create_default_context` lets the :mod:`!ssl` module choose security settings for a given purpose. .. versionchanged:: 3.6 @@ -1647,6 +1690,25 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.6 +.. method:: SSLContext.get_groups(*, include_aliases=False) + + Get a list of groups implemented for key agreement, taking into + account the current TLS :attr:`~SSLContext.minimum_version` and + :attr:`~SSLContext.maximum_version` values. For example:: + + >>> ctx = ssl.create_default_context() + >>> ctx.minimum_version = ssl.TLSVersion.TLSv1_3 + >>> ctx.maximum_version = ssl.TLSVersion.TLSv1_3 + >>> ctx.get_groups() # doctest: +SKIP + ['secp256r1', 'secp384r1', 'secp521r1', 'x25519', 'x448', ...] + + By default, this method returns only the preferred IANA names for the + available groups. However, if the ``include_aliases`` parameter is set to + :const:`True` this method will also return any associated aliases such as + the ECDH curve names supported in older versions of OpenSSL. + + .. versionadded:: 3.15 + .. method:: SSLContext.set_default_verify_paths() Load a set of default "certification authority" (CA) certificates from @@ -1656,23 +1718,79 @@ to speed up repeated connections from the same clients. provided as part of the operating system, though, it is likely to be configured properly. -.. method:: SSLContext.set_ciphers(ciphers) +.. method:: SSLContext.set_ciphers(ciphers, /) - Set the available ciphers for sockets created with this context. - It should be a string in the `OpenSSL cipher list format + Set the allowed ciphers for sockets created with this context when + connecting using TLS 1.2 and earlier. The *ciphers* argument should + be a string in the `OpenSSL cipher list format `_. + To set allowed TLS 1.3 ciphers, use :meth:`SSLContext.set_ciphersuites`. + If no cipher can be selected (because compile-time options or other configuration forbids use of all the specified ciphers), an :class:`SSLError` will be raised. .. note:: - when connected, the :meth:`SSLSocket.cipher` method of SSL sockets will - give the currently selected cipher. + When connected, the :meth:`SSLSocket.cipher` method of SSL sockets will + return details about the negotiated cipher. + +.. method:: SSLContext.set_ciphersuites(ciphersuites, /) + + Set the allowed ciphers for sockets created with this context when + connecting using TLS 1.3. The *ciphersuites* argument should be a + colon-separate string of TLS 1.3 cipher names. If no cipher can be + selected (because compile-time options or other configuration forbids + use of all the specified ciphers), an :class:`SSLError` will be raised. + + .. note:: + When connected, the :meth:`SSLSocket.cipher` method of SSL sockets will + return details about the negotiated cipher. + + .. versionadded:: 3.15 + +.. method:: SSLContext.set_groups(groups, /) + + Set the groups allowed for key agreement for sockets created with this + context. It should be a string in the `OpenSSL group list format + `_. + + .. note:: + + When connected, the :meth:`SSLSocket.group` method of SSL sockets will + return the group used for key agreement on that connection. + + .. versionadded:: 3.15 + +.. method:: SSLContext.set_client_sigalgs(sigalgs, /) + + Set the signature algorithms allowed for certificate-based client + authentication. It should be a string in the `OpenSSL client sigalgs + list format + `_. + + .. note:: + + When connected, the :meth:`SSLSocket.client_sigalg` method of SSL + sockets will return the signature algorithm used for performing + certificate-based client authentication on that connection. + + .. versionadded:: 3.15 + +.. method:: SSLContext.set_server_sigalgs(sigalgs, /) + + Set the signature algorithms allowed for the server to complete the TLS + handshake. It should be a string in the `OpenSSL sigalgs list format + `_. + + .. note:: + + When connected, the :meth:`SSLSocket.server_sigalg` method of SSL + sockets will return the signature algorithm used by the server to + complete the TLS handshake on that connection. - TLS 1.3 cipher suites cannot be disabled with - :meth:`~SSLContext.set_ciphers`. + .. versionadded:: 3.15 -.. method:: SSLContext.set_alpn_protocols(protocols) +.. method:: SSLContext.set_alpn_protocols(alpn_protocols) Specify which protocols the socket should advertise during the SSL/TLS handshake. It should be a list of ASCII strings, like ``['http/1.1', @@ -1686,7 +1804,7 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.5 -.. method:: SSLContext.set_npn_protocols(protocols) +.. method:: SSLContext.set_npn_protocols(npn_protocols) Specify which protocols the socket should advertise during the SSL/TLS handshake. It should be a list of strings, like ``['http/1.1', 'spdy/2']``, @@ -1753,7 +1871,7 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.7 -.. attribute:: SSLContext.set_servername_callback(server_name_callback) +.. method:: SSLContext.set_servername_callback(server_name_callback) This is a legacy API retained for backwards compatibility. When possible, you should use :attr:`sni_callback` instead. The given *server_name_callback* @@ -1767,7 +1885,7 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.4 -.. method:: SSLContext.load_dh_params(dhfile) +.. method:: SSLContext.load_dh_params(dhfile, /) Load the key generation parameters for Diffie-Hellman (DH) key exchange. Using DH key exchange improves forward secrecy at the expense of @@ -1780,7 +1898,7 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.3 -.. method:: SSLContext.set_ecdh_curve(curve_name) +.. method:: SSLContext.set_ecdh_curve(curve_name, /) Set the curve name for Elliptic Curve-based Diffie-Hellman (ECDH) key exchange. ECDH is significantly faster than regular DH while arguably @@ -1859,8 +1977,9 @@ to speed up repeated connections from the same clients. .. attribute:: SSLContext.sslsocket_class The return type of :meth:`SSLContext.wrap_socket`, defaults to - :class:`SSLSocket`. The attribute can be overridden on instance of class - in order to return a custom subclass of :class:`SSLSocket`. + :class:`SSLSocket`. The attribute can be assigned to on instances of + :class:`SSLContext` in order to return a custom subclass of + :class:`SSLSocket`. .. versionadded:: 3.7 @@ -1957,7 +2076,7 @@ to speed up repeated connections from the same clients. :attr:`~SSLContext.minimum_version` and :attr:`SSLContext.options` all affect the supported SSL and TLS versions of the context. The implementation does not prevent - invalid combination. For example a context with + invalid combinations. For example a context with :attr:`OP_NO_TLSv1_2` in :attr:`~SSLContext.options` and :attr:`~SSLContext.maximum_version` set to :attr:`TLSVersion.TLSv1_2` will not be able to establish a TLS 1.2 connection. @@ -2534,7 +2653,7 @@ thus several things you need to be aware of: as well. -Memory BIO Support +Memory BIO support ------------------ .. versionadded:: 3.5 @@ -2653,12 +2772,12 @@ purpose. It wraps an OpenSSL memory BIO (Basic IO) object: A boolean indicating whether the memory BIO is current at the end-of-file position. - .. method:: MemoryBIO.read(n=-1) + .. method:: MemoryBIO.read(n=-1, /) Read up to *n* bytes from the memory buffer. If *n* is not specified or negative, all bytes are returned. - .. method:: MemoryBIO.write(buf) + .. method:: MemoryBIO.write(buf, /) Write the bytes from *buf* to the memory BIO. The *buf* argument must be an object supporting the buffer protocol. @@ -2741,7 +2860,7 @@ This common check is automatically performed when .. versionchanged:: 3.7 Hostname matchings is now performed by OpenSSL. Python no longer uses - :func:`match_hostname`. + :func:`!match_hostname`. In server mode, if you want to authenticate your clients using the SSL layer (rather than using a higher-level authentication mechanism), you'll also have @@ -2760,11 +2879,11 @@ disabled by default. :: >>> client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_2 >>> client_context.maximum_version = ssl.TLSVersion.TLSv1_3 -The SSL context created above will only allow TLSv1.3 and later (if +The SSL client context created above will only allow TLSv1.2 and TLSv1.3 (if supported by your system) connections to a server. :const:`PROTOCOL_TLS_CLIENT` implies certificate validation and hostname checks by default. You have to load certificates into the context. @@ -2805,10 +2924,15 @@ TLS 1.3 The TLS 1.3 protocol behaves slightly differently than previous version of TLS/SSL. Some new TLS 1.3 features are not yet available. -- TLS 1.3 uses a disjunct set of cipher suites. All AES-GCM and - ChaCha20 cipher suites are enabled by default. The method - :meth:`SSLContext.set_ciphers` cannot enable or disable any TLS 1.3 - ciphers yet, but :meth:`SSLContext.get_ciphers` returns them. +- TLS 1.3 uses a disjunct set of cipher suites. All AES-GCM and ChaCha20 + cipher suites are enabled by default. To restrict which TLS 1.3 ciphers + are allowed, the :meth:`SSLContext.set_ciphersuites` method should be + called instead of :meth:`SSLContext.set_ciphers`, which only affects + ciphers in older TLS versions. The :meth:`SSLContext.get_ciphers` method + returns information about ciphers for both TLS 1.3 and earlier versions + and the method :meth:`SSLSocket.cipher` returns information about the + negotiated cipher for both TLS 1.3 and earlier versions once a connection + is established. - Session tickets are no longer sent as part of the initial handshake and are handled differently. :attr:`SSLSocket.session` and :class:`SSLSession` are not compatible with TLS 1.3. @@ -2817,7 +2941,7 @@ of TLS/SSL. Some new TLS 1.3 features are not yet available. process certificate requests while they send or receive application data from the server. - TLS 1.3 features like early data, deferred TLS client cert request, - signature algorithm configuration, and rekeying are not supported yet. + and rekeying are not supported yet. .. seealso:: @@ -2832,16 +2956,16 @@ of TLS/SSL. Some new TLS 1.3 features are not yet available. Steve Kent :rfc:`RFC 4086: Randomness Requirements for Security <4086>` - Donald E., Jeffrey I. Schiller + Donald E. Eastlake, Jeffrey I. Schiller, Steve Crocker :rfc:`RFC 5280: Internet X.509 Public Key Infrastructure Certificate and Certificate Revocation List (CRL) Profile <5280>` - D. Cooper + David Cooper et al. :rfc:`RFC 5246: The Transport Layer Security (TLS) Protocol Version 1.2 <5246>` - T. Dierks et. al. + Tim Dierks and Eric Rescorla. :rfc:`RFC 6066: Transport Layer Security (TLS) Extensions <6066>` - D. Eastlake + Donald E. Eastlake `IANA TLS: Transport Layer Security (TLS) Parameters `_ IANA diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index 8434b2e8c75cf42..5c5f1858ba44768 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -5,13 +5,11 @@ :synopsis: Utilities for interpreting the results of os.stat(), os.lstat() and os.fstat(). -.. sectionauthor:: Skip Montanaro - **Source code:** :source:`Lib/stat.py` -------------- -The :mod:`stat` module defines constants and functions for interpreting the +The :mod:`!stat` module defines constants and functions for interpreting the results of :func:`os.stat`, :func:`os.fstat` and :func:`os.lstat` (if they exist). For complete details about the :c:func:`stat`, :c:func:`!fstat` and :c:func:`!lstat` calls, consult the documentation for your system. @@ -19,7 +17,7 @@ exist). For complete details about the :c:func:`stat`, :c:func:`!fstat` and .. versionchanged:: 3.4 The stat module is backed by a C implementation. -The :mod:`stat` module defines the following functions to test for specific file +The :mod:`!stat` module defines the following functions to test for specific file types: @@ -493,3 +491,22 @@ constants, but are not an exhaustive list. IO_REPARSE_TAG_APPEXECLINK .. versionadded:: 3.8 + +On Linux, the following file attribute constants are available for use when +testing bits in the :attr:`~os.statx_result.stx_attributes` and +:attr:`~os.statx_result.stx_attributes_mask` members returned by +:func:`os.statx`. See the :manpage:`statx(2)` man page for more detail on the +meaning of these constants. + +.. data:: STATX_ATTR_COMPRESSED + STATX_ATTR_IMMUTABLE + STATX_ATTR_APPEND + STATX_ATTR_NODUMP + STATX_ATTR_ENCRYPTED + STATX_ATTR_AUTOMOUNT + STATX_ATTR_MOUNT_ROOT + STATX_ATTR_VERITY + STATX_ATTR_DAX + STATX_ATTR_WRITE_ATOMIC + + .. versionadded:: 3.15 diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 614f5b905a4a2eb..dba0e26787d9516 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -4,9 +4,6 @@ .. module:: statistics :synopsis: Mathematical statistics functions -.. moduleauthor:: Steven D'Aprano -.. sectionauthor:: Steven D'Aprano - .. versionadded:: 3.4 **Source code:** :source:`Lib/statistics.py` @@ -716,7 +713,7 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: covariance(x, y, /) - Return the sample covariance of two inputs *x* and *y*. Covariance + Return the sample covariance of two sequence inputs *x* and *y*. Covariance is a measure of the joint variability of two inputs. Both inputs must be of the same length (no less than two), otherwise @@ -742,7 +739,7 @@ However, for reading convenience, most of the examples show sorted sequences. Return the `Pearson's correlation coefficient `_ - for two inputs. Pearson's correlation coefficient *r* takes values + for two sequence inputs. Pearson's correlation coefficient *r* takes values between -1 and +1. It measures the strength and direction of a linear relationship. @@ -805,7 +802,7 @@ However, for reading convenience, most of the examples show sorted sequences. (it is equal to the difference between predicted and actual values of the dependent variable). - Both inputs must be of the same length (no less than two), and + Both inputs must be sequences of the same length (no less than two), and the independent variable *x* cannot be constant; otherwise a :exc:`StatisticsError` is raised. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 90683c0b00d78ab..a47e1ffb1a6afb1 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -46,8 +46,10 @@ Any object can be tested for truth value, for use in an :keyword:`if` or By default, an object is considered true unless its class defines either a :meth:`~object.__bool__` method that returns ``False`` or a :meth:`~object.__len__` method that -returns zero, when called with the object. [1]_ Here are most of the built-in -objects considered false: +returns zero, when called with the object. [1]_ If one of the methods raises an +exception when called, the exception is propagated and the object does +not have a truth value (for example, :data:`NotImplemented`). +Here are most of the built-in objects considered false: .. index:: single: None (Built-in object) @@ -164,7 +166,7 @@ This table summarizes the comparison operations: pair: object; numeric pair: objects; comparing -Objects of different types, except different numeric types, never compare equal. +Unless stated otherwise, objects of different types never compare equal. The ``==`` operator is always defined but for some object types (for example, class objects) is equivalent to :keyword:`is`. The ``<``, ``<=``, ``>`` and ``>=`` operators are only defined where they make sense; for example, they raise a @@ -263,9 +265,17 @@ The constructors :func:`int`, :func:`float`, and pair: operator; % (percent) pair: operator; ** +.. _stdtypes-mixed-arithmetic: + Python fully supports mixed arithmetic: when a binary arithmetic operator has -operands of different numeric types, the operand with the "narrower" type is -widened to that of the other, where integer is narrower than floating point. +operands of different built-in numeric types, the operand with the "narrower" +type is widened to that of the other: + +* If both arguments are complex numbers, no conversion is performed; +* if either argument is a complex or a floating-point number, the other is + converted to a floating-point number; +* otherwise, both must be integers and no conversion is necessary. + Arithmetic with complex and real operands is defined by the usual mathematical formula, for example:: @@ -1000,8 +1010,6 @@ operations have the same priority as the corresponding numeric operations. [3]_ pair: slice; operation pair: operator; in pair: operator; not in - single: count() (sequence method) - single: index() (sequence method) +--------------------------+--------------------------------+----------+ | Operation | Result | Notes | @@ -1018,7 +1026,7 @@ operations have the same priority as the corresponding numeric operations. [3]_ | ``s * n`` or | equivalent to adding *s* to | (2)(7) | | ``n * s`` | itself *n* times | | +--------------------------+--------------------------------+----------+ -| ``s[i]`` | *i*\ th item of *s*, origin 0 | (3)(9) | +| ``s[i]`` | *i*\ th item of *s*, origin 0 | (3)(8) | +--------------------------+--------------------------------+----------+ | ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) | +--------------------------+--------------------------------+----------+ @@ -1031,13 +1039,6 @@ operations have the same priority as the corresponding numeric operations. [3]_ +--------------------------+--------------------------------+----------+ | ``max(s)`` | largest item of *s* | | +--------------------------+--------------------------------+----------+ -| ``s.index(x[, i[, j]])`` | index of the first occurrence | \(8) | -| | of *x* in *s* (at or after | | -| | index *i* and before index *j*)| | -+--------------------------+--------------------------------+----------+ -| ``s.count(x)`` | total number of occurrences of | | -| | *x* in *s* | | -+--------------------------+--------------------------------+----------+ Sequences of the same type also support comparisons. In particular, tuples and lists are compared lexicographically by comparing corresponding elements. @@ -1100,11 +1101,14 @@ Notes: still ``0``. (4) - The slice of *s* from *i* to *j* is defined as the sequence of items with index - *k* such that ``i <= k < j``. If *i* or *j* is greater than ``len(s)``, use - ``len(s)``. If *i* is omitted or ``None``, use ``0``. If *j* is omitted or - ``None``, use ``len(s)``. If *i* is greater than or equal to *j*, the slice is - empty. + The slice of *s* from *i* to *j* is defined as the sequence of items with + index *k* such that ``i <= k < j``. + + * If *i* is omitted or ``None``, use ``0``. + * If *j* is omitted or ``None``, use ``len(s)``. + * If *i* or *j* is less than ``-len(s)``, use ``0``. + * If *i* or *j* is greater than ``len(s)``, use ``len(s)``. + * If *i* is greater than or equal to *j*, the slice is empty. (5) The slice of *s* from *i* to *j* with step *k* is defined as the sequence of @@ -1143,16 +1147,42 @@ Notes: concatenation or repetition. (8) - ``index`` raises :exc:`ValueError` when *x* is not found in *s*. - Not all implementations support passing the additional arguments *i* and *j*. - These arguments allow efficient searching of subsections of the sequence. Passing - the extra arguments is roughly equivalent to using ``s[i:j].index(x)``, only - without copying any data and with the returned index being relative to - the start of the sequence rather than the start of the slice. - -(9) An :exc:`IndexError` is raised if *i* is outside the sequence range. +.. rubric:: Sequence Methods + +Sequence types also support the following methods: + +.. method:: list.count(value, /) + range.count(value, /) + tuple.count(value, /) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.count(value, /) + + Return the total number of occurrences of *value* in *sequence*. + +.. method:: list.index(value[, start[, stop]]) + range.index(value[, start[, stop]]) + tuple.index(value[, start[, stop]]) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.index(value[, start[, stop]]) + + Return the index of the first occurrence of *value* in *sequence*. + + Raises :exc:`ValueError` if *value* is not found in *sequence*. + + The *start* or *stop* arguments allow for efficient searching + of subsections of the sequence, beginning at *start* and ending at *stop*. + This is roughly equivalent to ``start + sequence[start:stop].index(value)``, + only without copying any data. + + .. caution:: + Not all sequence types support passing the *start* and *stop* arguments. + .. _typesseq-immutable: @@ -1202,14 +1232,6 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). pair: subscript; assignment pair: slice; assignment pair: statement; del - single: append() (sequence method) - single: clear() (sequence method) - single: copy() (sequence method) - single: extend() (sequence method) - single: insert() (sequence method) - single: pop() (sequence method) - single: remove() (sequence method) - single: reverse() (sequence method) +------------------------------+--------------------------------+---------------------+ | Operation | Result | Notes | @@ -1233,39 +1255,14 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). | ``del s[i:j:k]`` | removes the elements of | | | | ``s[i:j:k]`` from the list | | +------------------------------+--------------------------------+---------------------+ -| ``s.append(x)`` | appends *x* to the end of the | | -| | sequence (same as | | -| | ``s[len(s):len(s)] = [x]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.clear()`` | removes all items from *s* | \(5) | -| | (same as ``del s[:]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.copy()`` | creates a shallow copy of *s* | \(5) | -| | (same as ``s[:]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.extend(t)`` or | extends *s* with the | | -| ``s += t`` | contents of *t* (for the | | +| ``s += t`` | extends *s* with the | | +| | contents of *t* (for the | | | | most part the same as | | | | ``s[len(s):len(s)] = t``) | | +------------------------------+--------------------------------+---------------------+ -| ``s *= n`` | updates *s* with its contents | \(6) | +| ``s *= n`` | updates *s* with its contents | \(2) | | | repeated *n* times | | +------------------------------+--------------------------------+---------------------+ -| ``s.insert(i, x)`` | inserts *x* into *s* at the | | -| | index given by *i* | | -| | (same as ``s[i:i] = [x]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.pop()`` or ``s.pop(i)`` | retrieves the item at *i* and | \(2) | -| | also removes it from *s* | | -+------------------------------+--------------------------------+---------------------+ -| ``s.remove(x)`` | removes the first item from | \(3) | -| | *s* where ``s[i]`` is equal to | | -| | *x* | | -+------------------------------+--------------------------------+---------------------+ -| ``s.reverse()`` | reverses the items of *s* in | \(4) | -| | place | | -+------------------------------+--------------------------------+---------------------+ - Notes: @@ -1273,32 +1270,105 @@ Notes: If *k* is not equal to ``1``, *t* must have the same length as the slice it is replacing. (2) - The optional argument *i* defaults to ``-1``, so that by default the last - item is removed and returned. + The value *n* is an integer, or an object implementing + :meth:`~object.__index__`. Zero and negative values of *n* clear + the sequence. Items in the sequence are not copied; they are referenced + multiple times, as explained for ``s * n`` under :ref:`typesseq-common`. -(3) - :meth:`remove` raises :exc:`ValueError` when *x* is not found in *s*. +.. rubric:: Mutable Sequence Methods -(4) - The :meth:`reverse` method modifies the sequence in place for economy of - space when reversing a large sequence. To remind users that it operates by - side effect, it does not return the reversed sequence. +Mutable sequence types also support the following methods: -(5) - :meth:`clear` and :meth:`!copy` are included for consistency with the - interfaces of mutable containers that don't support slicing operations - (such as :class:`dict` and :class:`set`). :meth:`!copy` is not part of the - :class:`collections.abc.MutableSequence` ABC, but most concrete - mutable sequence classes provide it. +.. method:: bytearray.append(value, /) + list.append(value, /) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.append(value, /) + + Append *value* to the end of the sequence. + This is equivalent to writing ``seq[len(seq):len(seq)] = [value]``. + +.. method:: bytearray.clear() + list.clear() + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.clear() .. versionadded:: 3.3 - :meth:`clear` and :meth:`!copy` methods. -(6) - The value *n* is an integer, or an object implementing - :meth:`~object.__index__`. Zero and negative values of *n* clear - the sequence. Items in the sequence are not copied; they are referenced - multiple times, as explained for ``s * n`` under :ref:`typesseq-common`. + Remove all items from *sequence*. + This is equivalent to writing ``del sequence[:]``. + +.. method:: bytearray.copy() + list.copy() + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.copy() + + .. versionadded:: 3.3 + + Create a shallow copy of *sequence*. + This is equivalent to writing ``sequence[:]``. + + .. hint:: The :meth:`!copy` method is not part of the + :class:`~collections.abc.MutableSequence` :class:`~abc.ABC`, + but most concrete mutable sequence types provide it. + +.. method:: bytearray.extend(iterable, /) + list.extend(iterable, /) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.extend(iterable, /) + + Extend *sequence* with the contents of *iterable*. + For the most part, this is the same as writing + ``seq[len(seq):len(seq)] = iterable``. + +.. method:: bytearray.insert(index, value, /) + list.insert(index, value, /) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.insert(index, value, /) + + Insert *value* into *sequence* at the given *index*. + This is equivalent to writing ``sequence[index:index] = [value]``. + +.. method:: bytearray.pop(index=-1, /) + list.pop(index=-1, /) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.pop(index=-1, /) + + Retrieve the item at *index* and also removes it from *sequence*. + By default, the last item in *sequence* is removed and returned. + +.. method:: bytearray.remove(value, /) + list.remove(value, /) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.remove(value, /) + + Remove the first item from *sequence* where ``sequence[i] == value``. + + Raises :exc:`ValueError` if *value* is not found in *sequence*. + +.. method:: bytearray.reverse() + list.reverse() + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.reverse() + + Reverse the items of *sequence* in place. + This method maintains economy of space when reversing a large sequence. + To remind users that it operates by side-effect, it returns ``None``. .. _typesseq-list: @@ -1312,7 +1382,7 @@ Lists are mutable sequences, typically used to store collections of homogeneous items (where the precise degree of similarity will vary by application). -.. class:: list([iterable]) +.. class:: list(iterable=(), /) Lists may be constructed in several ways: @@ -1333,6 +1403,8 @@ application). Many other operations also produce lists, including the :func:`sorted` built-in. + Lists are :ref:`generic ` over the types of their items. + Lists implement all of the :ref:`common ` and :ref:`mutable ` sequence operations. Lists also provide the following additional method: @@ -1379,6 +1451,11 @@ application). list appear empty for the duration, and raises :exc:`ValueError` if it can detect that the list has been mutated during a sort. +.. seealso:: + + For detailed information on thread-safety guarantees for :class:`list` + objects, see :ref:`thread-safety-list`. + .. _typesseq-tuple: @@ -1393,7 +1470,7 @@ built-in). Tuples are also used for cases where an immutable sequence of homogeneous data is needed (such as allowing storage in a :class:`set` or :class:`dict` instance). -.. class:: tuple([iterable]) +.. class:: tuple(iterable=(), /) Tuples may be constructed in a number of ways: @@ -1419,6 +1496,10 @@ homogeneous data is needed (such as allowing storage in a :class:`set` or Tuples implement all of the :ref:`common ` sequence operations. + Tuples are :ref:`generic ` over the types of their contents. + For more information, refer to + :ref:`the typing documentation on annotating tuples `. + For heterogeneous collections of data where access by name is clearer than access by index, :func:`collections.namedtuple` may be a more appropriate choice than a simple tuple object. @@ -1435,8 +1516,8 @@ The :class:`range` type represents an immutable sequence of numbers and is commonly used for looping a specific number of times in :keyword:`for` loops. -.. class:: range(stop) - range(start, stop[, step]) +.. class:: range(stop, /) + range(start, stop, step=1, /) The arguments to the range constructor must be integers (either built-in :class:`int` or any object that implements the :meth:`~object.__index__` special @@ -1696,8 +1777,10 @@ multiple fragments. .. index:: single: string; str (built-in class) -.. class:: str(object='') - str(object=b'', encoding='utf-8', errors='strict') +.. class:: str(*, encoding='utf-8', errors='strict') + str(object) + str(object, encoding, errors='strict') + str(object, *, errors) Return a :ref:`string ` version of *object*. If *object* is not provided, returns the empty string. Otherwise, the behavior of ``str()`` @@ -1783,15 +1866,23 @@ expression support in the :mod:`re` module). lowercase letter ``'ß'`` is equivalent to ``"ss"``. Since it is already lowercase, :meth:`lower` would do nothing to ``'ß'``; :meth:`casefold` converts it to ``"ss"``. + For example: - The casefolding algorithm is - `described in section 3.13 'Default Case Folding' of the Unicode Standard - `__. + .. doctest:: + + >>> 'straße'.lower() + 'straße' + >>> 'straße'.casefold() + 'strasse' + + The casefolding algorithm is `described in section 3.13.3 'Default Case + Folding' of the Unicode Standard + `__. .. versionadded:: 3.3 -.. method:: str.center(width[, fillchar]) +.. method:: str.center(width, fillchar=' ', /) Return centered in a string of length *width*. Padding is done using the specified *fillchar* (default is an ASCII space). The original string is @@ -1909,6 +2000,14 @@ expression support in the :mod:`re` module). Return the lowest index in the string where substring *sub* is found within the slice ``s[start:end]``. Optional arguments *start* and *end* are interpreted as in slice notation. Return ``-1`` if *sub* is not found. + For example:: + + >>> 'spam, spam, spam'.find('sp') + 0 + >>> 'spam, spam, spam'.find('sp', 5) + 6 + + See also :meth:`rfind` and :meth:`index`. .. note:: @@ -1927,10 +2026,16 @@ expression support in the :mod:`re` module). ``{}``. Each replacement field contains either the numeric index of a positional argument, or the name of a keyword argument. Returns a copy of the string where each replacement field is replaced with the string value of - the corresponding argument. + the corresponding argument. For example: + + .. doctest:: >>> "The sum of 1 + 2 is {0}".format(1+2) 'The sum of 1 + 2 is 3' + >>> "The sum of {a} + {b} is {answer}".format(answer=1+2, a=1, b=2) + 'The sum of 1 + 2 is 3' + >>> "{1} expects the {0} Inquisition!".format("Spanish", "Nobody") + 'Nobody expects the Spanish Inquisition!' See :ref:`formatstrings` for a description of the various formatting options that can be specified in format strings. @@ -1970,7 +2075,20 @@ expression support in the :mod:`re` module). .. method:: str.index(sub[, start[, end]]) Like :meth:`~str.find`, but raise :exc:`ValueError` when the substring is - not found. + not found. For example: + + .. doctest:: + + >>> 'spam, spam, spam'.index('spam') + 0 + >>> 'spam, spam, spam'.index('eggs') + Traceback (most recent call last): + File "", line 1, in + 'spam, spam, spam'.index('eggs') + ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + ValueError: substring not found + + See also :meth:`rindex`. .. method:: str.isalnum() @@ -1978,7 +2096,18 @@ expression support in the :mod:`re` module). Return ``True`` if all characters in the string are alphanumeric and there is at least one character, ``False`` otherwise. A character ``c`` is alphanumeric if one of the following returns ``True``: ``c.isalpha()``, ``c.isdecimal()``, - ``c.isdigit()``, or ``c.isnumeric()``. + ``c.isdigit()``, or ``c.isnumeric()``. For example: + + .. doctest:: + + >>> 'abc123'.isalnum() + True + >>> 'abc123!@#'.isalnum() + False + >>> ''.isalnum() + False + >>> ' '.isalnum() + False .. method:: str.isalpha() @@ -1989,14 +2118,33 @@ expression support in the :mod:`re` module). property being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is different from the `Alphabetic property defined in the section 4.10 'Letters, Alphabetic, and Ideographic' of the Unicode Standard - `_. + `__. + For example: + + .. doctest:: + + >>> 'Letters and spaces'.isalpha() + False + >>> 'LettersOnly'.isalpha() + True + >>> 'µ'.isalpha() # non-ASCII characters can be considered alphabetical too + True + + See :ref:`unicode-properties`. .. method:: str.isascii() Return ``True`` if the string is empty or all characters in the string are ASCII, ``False`` otherwise. - ASCII characters have code points in the range U+0000-U+007F. + ASCII characters have code points in the range U+0000-U+007F. For example: + + .. doctest:: + + >>> 'ASCII characters'.isascii() + True + >>> 'µ'.isascii() + False .. versionadded:: 3.7 @@ -2006,9 +2154,18 @@ expression support in the :mod:`re` module). Return ``True`` if all characters in the string are decimal characters and there is at least one character, ``False`` otherwise. Decimal characters are those that can be used to form - numbers in base 10, e.g. U+0660, ARABIC-INDIC DIGIT + numbers in base 10, such as U+0660, ARABIC-INDIC DIGIT ZERO. Formally a decimal character is a character in the Unicode - General Category "Nd". + General Category "Nd". For example: + + .. doctest:: + + >>> '0123456789'.isdecimal() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isdecimal() # Arabic-Indic digits zero to nine + True + >>> 'alphabetic'.isdecimal() + False .. method:: str.isdigit() @@ -2017,9 +2174,25 @@ expression support in the :mod:`re` module). character, ``False`` otherwise. Digits include decimal characters and digits that need special handling, such as the compatibility superscript digits. This covers digits which cannot be used to form numbers in base 10, - like the Kharosthi numbers. Formally, a digit is a character that has the + like the `Kharosthi numbers `__. + Formally, a digit is a character that has the property value Numeric_Type=Digit or Numeric_Type=Decimal. + For example: + + .. doctest:: + + >>> '0123456789'.isdigit() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isdigit() # Arabic-Indic digits zero to nine + True + >>> '⅕'.isdigit() # Vulgar fraction one fifth + False + >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() + (False, True, True) + + See also :meth:`isdecimal` and :meth:`isnumeric`. + .. method:: str.isidentifier() @@ -2054,6 +2227,20 @@ expression support in the :mod:`re` module). that have the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION ONE FIFTH. Formally, numeric characters are those with the property value Numeric_Type=Digit, Numeric_Type=Decimal or Numeric_Type=Numeric. + For example: + + .. doctest:: + + >>> '0123456789'.isnumeric() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-Indic digits zero to nine + True + >>> '⅕'.isnumeric() # Vulgar fraction one fifth + True + >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() + (False, True, True) + + See also :meth:`isdecimal` and :meth:`isdigit`. .. method:: str.isprintable() @@ -2072,17 +2259,43 @@ expression support in the :mod:`re` module). Nonprintable characters are those in group Separator or Other (Z or C), except the ASCII space. + For example: + + .. doctest:: + + >>> ''.isprintable(), ' '.isprintable() + (True, True) + >>> '\t'.isprintable(), '\n'.isprintable() + (False, False) + + See also :meth:`isspace`. + .. method:: str.isspace() Return ``True`` if there are only whitespace characters in the string and there is at least one character, ``False`` otherwise. + For example: + + .. doctest:: + + >>> ''.isspace() + False + >>> ' '.isspace() + True + >>> '\t\n'.isspace() # TAB and BREAK LINE + True + >>> '\u3000'.isspace() # IDEOGRAPHIC SPACE + True + A character is *whitespace* if in the Unicode character database (see :mod:`unicodedata`), either its general category is ``Zs`` ("Separator, space"), or its bidirectional class is one of ``WS``, ``B``, or ``S``. + See also :meth:`isprintable`. + .. method:: str.istitle() @@ -2090,6 +2303,19 @@ expression support in the :mod:`re` module). character, for example uppercase characters may only follow uncased characters and lowercase characters only cased ones. Return ``False`` otherwise. + For example: + + .. doctest:: + + >>> 'Spam, Spam, Spam'.istitle() + True + >>> 'spam, spam, spam'.istitle() + False + >>> 'SPAM, SPAM, SPAM'.istitle() + False + + See also :meth:`title`. + .. method:: str.isupper() @@ -2109,36 +2335,64 @@ expression support in the :mod:`re` module). .. _meth-str-join: -.. method:: str.join(iterable) +.. method:: str.join(iterable, /) Return a string which is the concatenation of the strings in *iterable*. A :exc:`TypeError` will be raised if there are any non-string values in *iterable*, including :class:`bytes` objects. The separator between - elements is the string providing this method. + elements is the string providing this method. For example: + .. doctest:: + + >>> ', '.join(['spam', 'spam', 'spam']) + 'spam, spam, spam' + >>> '-'.join('Python') + 'P-y-t-h-o-n' + + See also :meth:`split`. -.. method:: str.ljust(width[, fillchar]) + +.. method:: str.ljust(width, fillchar=' ', /) Return the string left justified in a string of length *width*. Padding is done using the specified *fillchar* (default is an ASCII space). The original string is returned if *width* is less than or equal to ``len(s)``. + For example: + + .. doctest:: + + >>> 'Python'.ljust(10) + 'Python ' + >>> 'Python'.ljust(10, '.') + 'Python....' + >>> 'Monty Python'.ljust(10, '.') + 'Monty Python' + + See also :meth:`rjust`. + .. method:: str.lower() Return a copy of the string with all the cased characters [4]_ converted to - lowercase. + lowercase. For example: - The lowercasing algorithm used is - `described in section 3.13 'Default Case Folding' of the Unicode Standard - `__. + .. doctest:: + >>> 'Lower Method Example'.lower() + 'lower method example' -.. method:: str.lstrip([chars]) + The lowercasing algorithm used is `described in section 3.13.2 'Default Case + Conversion' of the Unicode Standard + `__. + + +.. method:: str.lstrip(chars=None, /) Return a copy of the string with leading characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted - or ``None``, the *chars* argument defaults to removing whitespace. The *chars* + or ``None``, the *chars* argument defaults to removing whitespace, that is + characters for which :meth:`str.isspace` is true. The *chars* argument is not a prefix; rather, all combinations of its values are stripped:: >>> ' spacious '.lstrip() @@ -2155,7 +2409,8 @@ expression support in the :mod:`re` module). 'three!' -.. staticmethod:: str.maketrans(x[, y[, z]]) +.. staticmethod:: str.maketrans(dict, /) + str.maketrans(from, to, remove='', /) This static method returns a translation table usable for :meth:`str.translate`. @@ -2165,24 +2420,43 @@ expression support in the :mod:`re` module). converted to ordinals. If there are two arguments, they must be strings of equal length, and in the - resulting dictionary, each character in x will be mapped to the character at - the same position in y. If there is a third argument, it must be a string, + resulting dictionary, each character in *from* will be mapped to the character at + the same position in *to*. If there is a third argument, it must be a string, whose characters will be mapped to ``None`` in the result. + .. versionchanged:: 3.15 + + *dict* can now be a :class:`frozendict`. -.. method:: str.partition(sep) + +.. method:: str.partition(sep, /) Split the string at the first occurrence of *sep*, and return a 3-tuple containing the part before the separator, the separator itself, and the part after the separator. If the separator is not found, return a 3-tuple containing the string itself, followed by two empty strings. + For example: + + .. doctest:: + + >>> 'Monty Python'.partition(' ') + ('Monty', ' ', 'Python') + >>> "Monty Python's Flying Circus".partition(' ') + ('Monty', ' ', "Python's Flying Circus") + >>> 'Monty Python'.partition('-') + ('Monty Python', '', '') + + See also :meth:`rpartition`. + .. method:: str.removeprefix(prefix, /) If the string starts with the *prefix* string, return ``string[len(prefix):]``. Otherwise, return a copy of the original - string:: + string: + + .. doctest:: >>> 'TestHook'.removeprefix('Test') 'Hook' @@ -2191,12 +2465,16 @@ expression support in the :mod:`re` module). .. versionadded:: 3.9 + See also :meth:`removesuffix` and :meth:`startswith`. + .. method:: str.removesuffix(suffix, /) If the string ends with the *suffix* string and that *suffix* is not empty, return ``string[:-len(suffix)]``. Otherwise, return a copy of the - original string:: + original string: + + .. doctest:: >>> 'MiscTests'.removesuffix('Tests') 'Misc' @@ -2205,12 +2483,22 @@ expression support in the :mod:`re` module). .. versionadded:: 3.9 + See also :meth:`removeprefix` and :meth:`endswith`. -.. method:: str.replace(old, new, count=-1) + +.. method:: str.replace(old, new, /, count=-1) Return a copy of the string with all occurrences of substring *old* replaced by *new*. If *count* is given, only the first *count* occurrences are replaced. If *count* is not specified or ``-1``, then all occurrences are replaced. + For example: + + .. doctest:: + + >>> 'spam, spam, spam'.replace('spam', 'eggs') + 'eggs, eggs, eggs' + >>> 'spam, spam, spam'.replace('spam', 'eggs', 1) + 'eggs, spam, spam' .. versionchanged:: 3.13 *count* is now supported as a keyword argument. @@ -2221,28 +2509,78 @@ expression support in the :mod:`re` module). Return the highest index in the string where substring *sub* is found, such that *sub* is contained within ``s[start:end]``. Optional arguments *start* and *end* are interpreted as in slice notation. Return ``-1`` on failure. + For example: + + .. doctest:: + + >>> 'spam, spam, spam'.rfind('sp') + 12 + >>> 'spam, spam, spam'.rfind('sp', 0, 10) + 6 + + See also :meth:`find` and :meth:`rindex`. .. method:: str.rindex(sub[, start[, end]]) Like :meth:`rfind` but raises :exc:`ValueError` when the substring *sub* is not found. + For example: + + .. doctest:: + + >>> 'spam, spam, spam'.rindex('spam') + 12 + >>> 'spam, spam, spam'.rindex('eggs') + Traceback (most recent call last): + File "", line 1, in + 'spam, spam, spam'.rindex('eggs') + ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + ValueError: substring not found + + See also :meth:`index` and :meth:`find`. -.. method:: str.rjust(width[, fillchar]) +.. method:: str.rjust(width, fillchar=' ', /) Return the string right justified in a string of length *width*. Padding is done using the specified *fillchar* (default is an ASCII space). The original string is returned if *width* is less than or equal to ``len(s)``. + For example: + + .. doctest:: + + >>> 'Python'.rjust(10) + ' Python' + >>> 'Python'.rjust(10, '.') + '....Python' + >>> 'Monty Python'.rjust(10, '.') + 'Monty Python' + + See also :meth:`ljust` and :meth:`zfill`. + -.. method:: str.rpartition(sep) +.. method:: str.rpartition(sep, /) Split the string at the last occurrence of *sep*, and return a 3-tuple containing the part before the separator, the separator itself, and the part after the separator. If the separator is not found, return a 3-tuple containing two empty strings, followed by the string itself. + For example: + + .. doctest:: + + >>> 'Monty Python'.rpartition(' ') + ('Monty', ' ', 'Python') + >>> "Monty Python's Flying Circus".rpartition(' ') + ("Monty Python's Flying", ' ', 'Circus') + >>> 'Monty Python'.rpartition('-') + ('', '', 'Monty Python') + + See also :meth:`partition`. + .. method:: str.rsplit(sep=None, maxsplit=-1) @@ -2253,19 +2591,23 @@ expression support in the :mod:`re` module). :meth:`split` which is described in detail below. -.. method:: str.rstrip([chars]) +.. method:: str.rstrip(chars=None, /) Return a copy of the string with trailing characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted - or ``None``, the *chars* argument defaults to removing whitespace. The *chars* - argument is not a suffix; rather, all combinations of its values are stripped:: + or ``None``, the *chars* argument defaults to removing whitespace, that is + characters for which :meth:`str.isspace` is true. The *chars* + argument is not a suffix; rather, all combinations of its values are stripped. + For example: + + .. doctest:: >>> ' spacious '.rstrip() ' spacious' >>> 'mississippi'.rstrip('ipz') 'mississ' - See :meth:`str.removesuffix` for a method that will remove a single suffix + See :meth:`removesuffix` for a method that will remove a single suffix string rather than all of a set of characters. For example:: >>> 'Monty Python'.rstrip(' Python') @@ -2273,6 +2615,9 @@ expression support in the :mod:`re` module). >>> 'Monty Python'.removesuffix(' Python') 'Monty' + See also :meth:`strip`. + + .. method:: str.split(sep=None, maxsplit=-1) Return a list of the words in the string, using *sep* as the delimiter @@ -2288,7 +2633,9 @@ expression support in the :mod:`re` module). :func:`re.split`). Splitting an empty string with a specified separator returns ``['']``. - For example:: + For example: + + .. doctest:: >>> '1,2,3'.split(',') ['1', '2', '3'] @@ -2306,7 +2653,9 @@ expression support in the :mod:`re` module). string or a string consisting of just whitespace with a ``None`` separator returns ``[]``. - For example:: + For example: + + .. doctest:: >>> '1 2 3'.split() ['1', '2', '3'] @@ -2318,7 +2667,9 @@ expression support in the :mod:`re` module). If *sep* is not specified or is ``None`` and *maxsplit* is ``0``, only leading runs of consecutive whitespace are considered. - For example:: + For example: + + .. doctest:: >>> "".split(None, 0) [] @@ -2327,6 +2678,8 @@ expression support in the :mod:`re` module). >>> " foo ".split(maxsplit=0) ['foo '] + See also :meth:`join` and :meth:`rsplit`. + .. index:: single: universal newlines; str.splitlines method @@ -2401,14 +2754,31 @@ expression support in the :mod:`re` module). test string beginning at that position. With optional *end*, stop comparing string at that position. + For example: + + .. doctest:: + + >>> 'Python'.startswith('Py') + True + >>> 'a tuple of prefixes'.startswith(('at', 'a')) + True + >>> 'Python is amazing'.startswith('is', 7) + True + + See also :meth:`endswith` and :meth:`removeprefix`. + -.. method:: str.strip([chars]) +.. method:: str.strip(chars=None, /) Return a copy of the string with the leading and trailing characters removed. The *chars* argument is a string specifying the set of characters to be removed. - If omitted or ``None``, the *chars* argument defaults to removing whitespace. - The *chars* argument is not a prefix or suffix; rather, all combinations of its - values are stripped:: + If omitted or ``None``, the *chars* argument defaults to removing whitespace, + that is characters for which :meth:`str.isspace` is true. The *chars* argument + is not a prefix or suffix; rather, all combinations of its values are stripped. + + For example: + + .. doctest:: >>> ' spacious '.strip() 'spacious' @@ -2419,18 +2789,37 @@ expression support in the :mod:`re` module). from the string. Characters are removed from the leading end until reaching a string character that is not contained in the set of characters in *chars*. A similar action takes place on the trailing end. - For example:: + + For example: + + .. doctest:: >>> comment_string = '#....... Section 3.2.1 Issue #32 .......' >>> comment_string.strip('.#! ') 'Section 3.2.1 Issue #32' + See also :meth:`rstrip`. + .. method:: str.swapcase() Return a copy of the string with uppercase characters converted to lowercase and - vice versa. Note that it is not necessarily true that - ``s.swapcase().swapcase() == s``. + vice versa. For example: + + .. doctest:: + + >>> 'Hello World'.swapcase() + 'hELLO wORLD' + + Note that it is not necessarily true that ``s.swapcase().swapcase() == s``. + For example: + + .. doctest:: + + >>> 'straße'.swapcase().swapcase() + 'strasse' + + See also :meth:`str.lower` and :meth:`str.upper`. .. method:: str.title() @@ -2466,8 +2855,10 @@ expression support in the :mod:`re` module). >>> titlecase("they're bill's friends.") "They're Bill's Friends." + See also :meth:`istitle`. + -.. method:: str.translate(table) +.. method:: str.translate(table, /) Return a copy of the string in which each character has been mapped through the given translation table. The table must be an object that implements @@ -2481,6 +2872,14 @@ expression support in the :mod:`re` module). You can use :meth:`str.maketrans` to create a translation map from character-to-character mappings in different formats. + The following example uses a mapping to replace ``'a'`` with ``'X'``, + ``'b'`` with ``'Y'``, and delete ``'c'``: + + .. doctest:: + + >>> 'abc123'.translate({ord('a'): 'X', ord('b'): 'Y', ord('c'): None}) + 'XY123' + See also the :mod:`codecs` module for a more flexible approach to custom character mappings. @@ -2493,12 +2892,12 @@ expression support in the :mod:`re` module). character(s) is not "Lu" (Letter, uppercase), but e.g. "Lt" (Letter, titlecase). - The uppercasing algorithm used is - `described in section 3.13 'Default Case Folding' of the Unicode Standard - `__. + The uppercasing algorithm used is `described in section 3.13.2 'Default Case + Conversion' of the Unicode Standard + `__. -.. method:: str.zfill(width) +.. method:: str.zfill(width, /) Return a copy of the string left filled with ASCII ``'0'`` digits to make a string of length *width*. A leading sign prefix (``'+'``/``'-'``) @@ -2506,13 +2905,17 @@ expression support in the :mod:`re` module). than before. The original string is returned if *width* is less than or equal to ``len(s)``. - For example:: + For example: + + .. doctest:: >>> "42".zfill(5) '00042' >>> "-42".zfill(5) '-0042' + See also :meth:`rjust`. + .. index:: single: ! formatted string literal @@ -2528,6 +2931,8 @@ expression support in the :mod:`re` module). single: : (colon); in formatted string literal single: = (equals); for help in debugging using string literals +.. _stdtypes-fstrings: + Formatted String Literals (f-strings) ------------------------------------- @@ -2536,123 +2941,147 @@ Formatted String Literals (f-strings) The :keyword:`await` and :keyword:`async for` can be used in expressions within f-strings. .. versionchanged:: 3.8 - Added the debugging operator (``=``) + Added the debug specifier (``=``) .. versionchanged:: 3.12 Many restrictions on expressions within f-strings have been removed. Notably, nested strings, comments, and backslashes are now permitted. An :dfn:`f-string` (formally a :dfn:`formatted string literal`) is a string literal that is prefixed with ``f`` or ``F``. -This type of string literal allows embedding arbitrary Python expressions -within *replacement fields*, which are delimited by curly brackets (``{}``). -These expressions are evaluated at runtime, similarly to :meth:`str.format`, -and are converted into regular :class:`str` objects. -For example: +This type of string literal allows embedding the results of arbitrary Python +expressions within *replacement fields*, which are delimited by curly +brackets (``{}``). +Each replacement field must contain an expression, optionally followed by: -.. doctest:: +* a *debug specifier* -- an equal sign (``=``); +* a *conversion specifier* -- ``!s``, ``!r`` or ``!a``; and/or +* a *format specifier* prefixed with a colon (``:``). - >>> who = 'nobody' - >>> nationality = 'Spanish' - >>> f'{who.title()} expects the {nationality} Inquisition!' - 'Nobody expects the Spanish Inquisition!' +See the :ref:`Lexical Analysis section on f-strings ` for details +on the syntax of these fields. -It is also possible to use a multi line f-string: +Debug specifier +^^^^^^^^^^^^^^^ -.. doctest:: +.. versionadded:: 3.8 - >>> f'''This is a string - ... on two lines''' - 'This is a string\non two lines' +If a debug specifier -- an equal sign (``=``) -- appears after the replacement +field expression, the resulting f-string will contain the expression's source, +the equal sign, and the value of the expression. +This is often useful for debugging:: -A single opening curly bracket, ``'{'``, marks a *replacement field* that -can contain any Python expression: + >>> number = 14.3 + >>> f'{number=}' + 'number=14.3' -.. doctest:: - - >>> nationality = 'Spanish' - >>> f'The {nationality} Inquisition!' - 'The Spanish Inquisition!' - -To include a literal ``{`` or ``}``, use a double bracket: - -.. doctest:: +Whitespace before, inside and after the expression, as well as whitespace +after the equal sign, is significant --- it is retained in the result:: - >>> x = 42 - >>> f'{{x}} is {x}' - '{x} is 42' + >>> f'{ number - 4 = }' + ' number - 4 = 10.3' -Functions can also be used, and :ref:`format specifiers `: -.. doctest:: - - >>> from math import sqrt - >>> f'√2 \N{ALMOST EQUAL TO} {sqrt(2):.5f}' - '√2 ≈ 1.41421' +Conversion specifier +^^^^^^^^^^^^^^^^^^^^ -Any non-string expression is converted using :func:`str`, by default: - -.. doctest:: +By default, the value of a replacement field expression is converted to +a string using :func:`str`:: >>> from fractions import Fraction - >>> f'{Fraction(1, 3)}' + >>> one_third = Fraction(1, 3) + >>> f'{one_third}' '1/3' -To use an explicit conversion, use the ``!`` (exclamation mark) operator, -followed by any of the valid formats, which are: +When a debug specifier but no format specifier is used, the default conversion +instead uses :func:`repr`:: -========== ============== -Conversion Meaning -========== ============== -``!a`` :func:`ascii` -``!r`` :func:`repr` -``!s`` :func:`str` -========== ============== + >>> f'{one_third = }' + 'one_third = Fraction(1, 3)' -For example: +The conversion can be specified explicitly using one of these specifiers: -.. doctest:: +* ``!s`` for :func:`str` +* ``!r`` for :func:`repr` +* ``!a`` for :func:`ascii` - >>> from fractions import Fraction - >>> f'{Fraction(1, 3)!s}' +For example:: + + >>> str(one_third) '1/3' - >>> f'{Fraction(1, 3)!r}' + >>> repr(one_third) 'Fraction(1, 3)' - >>> question = '¿Dónde está el Presidente?' - >>> print(f'{question!a}') - '\xbfD\xf3nde est\xe1 el Presidente?' - -While debugging it may be helpful to see both the expression and its value, -by using the equals sign (``=``) after the expression. -This preserves spaces within the brackets, and can be used with a converter. -By default, the debugging operator uses the :func:`repr` (``!r``) conversion. -For example: -.. doctest:: + >>> f'{one_third!s} is {one_third!r}' + '1/3 is Fraction(1, 3)' - >>> from fractions import Fraction - >>> calculation = Fraction(1, 3) - >>> f'{calculation=}' - 'calculation=Fraction(1, 3)' - >>> f'{calculation = }' - 'calculation = Fraction(1, 3)' - >>> f'{calculation = !s}' - 'calculation = 1/3' - -Once the output has been evaluated, it can be formatted using a -:ref:`format specifier ` following a colon (``':'``). -After the expression has been evaluated, and possibly converted to a string, -the :meth:`!__format__` method of the result is called with the format specifier, -or the empty string if no format specifier is given. -The formatted result is then used as the final value for the replacement field. -For example: + >>> string = "¡kočka 😸!" + >>> ascii(string) + "'\\xa1ko\\u010dka \\U0001f638!'" -.. doctest:: + >>> f'{string = !a}' + "string = '\\xa1ko\\u010dka \\U0001f638!'" + + +Format specifier +^^^^^^^^^^^^^^^^ + +After the expression has been evaluated, and possibly converted using an +explicit conversion specifier, it is formatted using the :func:`format` function. +If the replacement field includes a *format specifier* introduced by a colon +(``:``), the specifier is passed to :func:`!format` as the second argument. +The result of :func:`!format` is then used as the final value for the +replacement field. For example:: >>> from fractions import Fraction - >>> f'{Fraction(1, 7):.6f}' - '0.142857' - >>> f'{Fraction(1, 7):_^+10}' - '___+1/7___' + >>> one_third = Fraction(1, 3) + >>> f'{one_third:.6f}' + '0.333333' + >>> f'{one_third:_^+10}' + '___+1/3___' + >>> >>> f'{one_third!r:_^20}' + '___Fraction(1, 3)___' + >>> f'{one_third = :~>10}~' + 'one_third = ~~~~~~~1/3~' + +.. _stdtypes-tstrings: + +Template String Literals (t-strings) +------------------------------------ + +An :dfn:`t-string` (formally a :dfn:`template string literal`) is +a string literal that is prefixed with ``t`` or ``T``. + +These strings follow the same syntax and evaluation rules as +:ref:`formatted string literals `, +with for the following differences: + +* Rather than evaluating to a ``str`` object, template string literals evaluate + to a :class:`string.templatelib.Template` object. + +* The :func:`format` protocol is not used. + Instead, the format specifier and conversions (if any) are passed to + a new :class:`~string.templatelib.Interpolation` object that is created + for each evaluated expression. + It is up to code that processes the resulting :class:`~string.templatelib.Template` + object to decide how to handle format specifiers and conversions. + +* Format specifiers containing nested replacement fields are evaluated eagerly, + prior to being passed to the :class:`~string.templatelib.Interpolation` object. + For instance, an interpolation of the form ``{amount:.{precision}f}`` will + evaluate the inner expression ``{precision}`` to determine the value of the + ``format_spec`` attribute. + If ``precision`` were to be ``2``, the resulting format specifier + would be ``'.2f'``. + +* When the equals sign ``'='`` is provided in an interpolation expression, + the text of the expression is appended to the literal string that precedes + the relevant interpolation. + This includes the equals sign and any surrounding whitespace. + The :class:`!Interpolation` instance for the expression will be created as + normal, except that :attr:`~string.templatelib.Interpolation.conversion` will + be set to '``r``' (:func:`repr`) by default. + If an explicit conversion or format specifier are provided, + this will override the default behaviour. .. _old-string-formatting: @@ -2673,9 +3102,10 @@ For example: The formatting operations described here exhibit a variety of quirks that lead to a number of common errors (such as failing to display tuples and - dictionaries correctly). Using the newer :ref:`formatted string literals - `, the :meth:`str.format` interface, or :ref:`template strings - ($-strings) ` may help avoid these errors. + dictionaries correctly). + + Using :ref:`formatted string literals `, the :meth:`str.format` + interface, or :class:`string.Template` may help avoid these errors. Each of these alternatives provides their own trade-offs and benefits of simplicity, flexibility, and/or extensibility. @@ -2815,6 +3245,10 @@ The conversion types are: | | character in the result. | | +------------+-----------------------------------------------------+-------+ +For floating-point formats, the result should be correctly rounded to a given +precision ``p`` of digits after the decimal point. The rounding mode matches +that of the :func:`round` builtin. + Notes: (1) @@ -2889,7 +3323,8 @@ binary protocols are based on the ASCII text encoding, bytes objects offer several methods that are only valid when working with ASCII compatible data and are closely related to string objects in a variety of other ways. -.. class:: bytes([source[, encoding[, errors]]]) +.. class:: bytes(source=b'') + bytes(source, encoding, errors='strict') Firstly, the syntax for bytes literals is largely the same as that for string literals, except that a ``b`` prefix is added: @@ -2929,7 +3364,7 @@ data and are closely related to string objects in a variety of other ways. numbers are a commonly used format for describing binary data. Accordingly, the bytes type has an additional class method to read data in that format: - .. classmethod:: fromhex(string) + .. classmethod:: fromhex(string, /) This :class:`bytes` class method returns a bytes object, decoding the given string object. The string must contain two hexadecimal digits per @@ -2949,7 +3384,8 @@ data and are closely related to string objects in a variety of other ways. A reverse conversion function exists to transform a bytes object into its hexadecimal representation. - .. method:: hex([sep[, bytes_per_sep]]) + .. method:: hex(*, bytes_per_sep=1) + hex(sep, bytes_per_sep=1) Return a string object containing two hexadecimal digits for each byte in the instance. @@ -2998,7 +3434,8 @@ Bytearray Objects :class:`bytearray` objects are a mutable counterpart to :class:`bytes` objects. -.. class:: bytearray([source[, encoding[, errors]]]) +.. class:: bytearray(source=b'') + bytearray(source, encoding, errors='strict') There is no dedicated literal syntax for bytearray objects, instead they are always created by calling the constructor: @@ -3018,7 +3455,7 @@ objects. numbers are a commonly used format for describing binary data. Accordingly, the bytearray type has an additional class method to read data in that format: - .. classmethod:: fromhex(string) + .. classmethod:: fromhex(string, /) This :class:`bytearray` class method returns bytearray object, decoding the given string object. The string must contain two hexadecimal digits @@ -3038,7 +3475,8 @@ objects. A reverse conversion function exists to transform a bytearray object into its hexadecimal representation. - .. method:: hex([sep[, bytes_per_sep]]) + .. method:: hex(*, bytes_per_sep=1) + hex(sep, bytes_per_sep=1) Return a string object containing two hexadecimal digits for each byte in the instance. @@ -3053,7 +3491,7 @@ objects. optional *sep* and *bytes_per_sep* parameters to insert separators between bytes in the hex output. - .. method:: resize(size) + .. method:: resize(size, /) Resize the :class:`bytearray` to contain *size* bytes. *size* must be greater than or equal to 0. @@ -3085,6 +3523,30 @@ objects. .. versionadded:: 3.14 + .. method:: take_bytes(n=None, /) + + Remove the first *n* bytes from the bytearray and return them as an immutable + :class:`bytes`. + By default (if *n* is ``None``), return all bytes and clear the bytearray. + + If *n* is negative, index from the end and take the first :func:`len` + plus *n* bytes. If *n* is out of bounds, raise :exc:`IndexError`. + + Taking less than the full length will leave remaining bytes in the + :class:`bytearray`, which requires a copy. If the remaining bytes should be + discarded, use :func:`~bytearray.resize` or :keyword:`del` to truncate + then :func:`~bytearray.take_bytes` without a size. + + .. impl-detail:: + + Taking all bytes is a zero-copy operation. + + .. versionadded:: 3.15 + + See the :ref:`What's New ` entry for + common code patterns which can be optimized with + :meth:`bytearray.take_bytes`. + Since bytearray objects are sequences of integers (akin to a list), for a bytearray object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytearray object of length 1. (This contrasts with text strings, where @@ -3095,6 +3557,11 @@ The representation of bytearray objects uses the bytes literal format ``bytearray([46, 46, 46])``. You can always convert a bytearray object into a list of integers using ``list(b)``. +.. seealso:: + + For detailed information on thread-safety guarantees for :class:`bytearray` + objects, see :ref:`thread-safety-bytearray`. + .. _bytes-methods: @@ -3276,8 +3743,8 @@ arbitrary binary data. Also accept an integer in the range 0 to 255 as the subsequence. -.. method:: bytes.join(iterable) - bytearray.join(iterable) +.. method:: bytes.join(iterable, /) + bytearray.join(iterable, /) Return a bytes or bytearray object which is the concatenation of the binary data sequences in *iterable*. A :exc:`TypeError` will be raised @@ -3287,8 +3754,8 @@ arbitrary binary data. bytearray object providing this method. -.. staticmethod:: bytes.maketrans(from, to) - bytearray.maketrans(from, to) +.. staticmethod:: bytes.maketrans(from, to, /) + bytearray.maketrans(from, to, /) This static method returns a translation table usable for :meth:`bytes.translate` that will map each character in *from* into the @@ -3298,8 +3765,8 @@ arbitrary binary data. .. versionadded:: 3.1 -.. method:: bytes.partition(sep) - bytearray.partition(sep) +.. method:: bytes.partition(sep, /) + bytearray.partition(sep, /) Split the sequence at the first occurrence of *sep*, and return a 3-tuple containing the part before the separator, the separator itself or its @@ -3311,12 +3778,13 @@ arbitrary binary data. The separator to search for may be any :term:`bytes-like object`. -.. method:: bytes.replace(old, new[, count]) - bytearray.replace(old, new[, count]) +.. method:: bytes.replace(old, new, /, count=-1) + bytearray.replace(old, new, /, count=-1) Return a copy of the sequence with all occurrences of subsequence *old* - replaced by *new*. If the optional argument *count* is given, only the - first *count* occurrences are replaced. + replaced by *new*. If *count* is given, only the first *count* occurrences + are replaced. If *count* is not specified or ``-1``, then all occurrences + are replaced. The subsequence to search for and its replacement may be any :term:`bytes-like object`. @@ -3326,6 +3794,9 @@ arbitrary binary data. The bytearray version of this method does *not* operate in place - it always produces a new object, even if no changes were made. + .. versionchanged:: 3.15 + *count* is now supported as a keyword argument. + .. method:: bytes.rfind(sub[, start[, end]]) bytearray.rfind(sub[, start[, end]]) @@ -3355,8 +3826,8 @@ arbitrary binary data. Also accept an integer in the range 0 to 255 as the subsequence. -.. method:: bytes.rpartition(sep) - bytearray.rpartition(sep) +.. method:: bytes.rpartition(sep, /) + bytearray.rpartition(sep, /) Split the sequence at the last occurrence of *sep*, and return a 3-tuple containing the part before the separator, the separator itself or its @@ -3406,8 +3877,8 @@ with arbitrary binary data by passing appropriate arguments. Note that all of the bytearray methods in this section do *not* operate in place, and instead produce new objects. -.. method:: bytes.center(width[, fillbyte]) - bytearray.center(width[, fillbyte]) +.. method:: bytes.center(width, fillbyte=b' ', /) + bytearray.center(width, fillbyte=b' ', /) Return a copy of the object centered in a sequence of length *width*. Padding is done using the specified *fillbyte* (default is an ASCII @@ -3420,8 +3891,8 @@ produce new objects. it always produces a new object, even if no changes were made. -.. method:: bytes.ljust(width[, fillbyte]) - bytearray.ljust(width[, fillbyte]) +.. method:: bytes.ljust(width, fillbyte=b' ', /) + bytearray.ljust(width, fillbyte=b' ', /) Return a copy of the object left justified in a sequence of length *width*. Padding is done using the specified *fillbyte* (default is an ASCII @@ -3434,14 +3905,13 @@ produce new objects. it always produces a new object, even if no changes were made. -.. method:: bytes.lstrip([chars]) - bytearray.lstrip([chars]) +.. method:: bytes.lstrip(bytes=None, /) + bytearray.lstrip(bytes=None, /) Return a copy of the sequence with specified leading bytes removed. The - *chars* argument is a binary sequence specifying the set of byte values to - be removed - the name refers to the fact this method is usually used with - ASCII characters. If omitted or ``None``, the *chars* argument defaults - to removing ASCII whitespace. The *chars* argument is not a prefix; + *bytes* argument is a binary sequence specifying the set of byte values to + be removed. If omitted or ``None``, the *bytes* argument defaults + to removing ASCII whitespace. The *bytes* argument is not a prefix; rather, all combinations of its values are stripped:: >>> b' spacious '.lstrip() @@ -3465,8 +3935,8 @@ produce new objects. it always produces a new object, even if no changes were made. -.. method:: bytes.rjust(width[, fillbyte]) - bytearray.rjust(width[, fillbyte]) +.. method:: bytes.rjust(width, fillbyte=b' ', /) + bytearray.rjust(width, fillbyte=b' ', /) Return a copy of the object right justified in a sequence of length *width*. Padding is done using the specified *fillbyte* (default is an ASCII @@ -3490,14 +3960,13 @@ produce new objects. :meth:`split` which is described in detail below. -.. method:: bytes.rstrip([chars]) - bytearray.rstrip([chars]) +.. method:: bytes.rstrip(bytes=None, /) + bytearray.rstrip(bytes=None, /) Return a copy of the sequence with specified trailing bytes removed. The - *chars* argument is a binary sequence specifying the set of byte values to - be removed - the name refers to the fact this method is usually used with - ASCII characters. If omitted or ``None``, the *chars* argument defaults to - removing ASCII whitespace. The *chars* argument is not a suffix; rather, + *bytes* argument is a binary sequence specifying the set of byte values to + be removed. If omitted or ``None``, the *bytes* argument defaults to + removing ASCII whitespace. The *bytes* argument is not a suffix; rather, all combinations of its values are stripped:: >>> b' spacious '.rstrip() @@ -3567,14 +4036,13 @@ produce new objects. [b'1', b'2', b'3'] -.. method:: bytes.strip([chars]) - bytearray.strip([chars]) +.. method:: bytes.strip(bytes=None, /) + bytearray.strip(bytes=None, /) Return a copy of the sequence with specified leading and trailing bytes - removed. The *chars* argument is a binary sequence specifying the set of - byte values to be removed - the name refers to the fact this method is - usually used with ASCII characters. If omitted or ``None``, the *chars* - argument defaults to removing ASCII whitespace. The *chars* argument is + removed. The *bytes* argument is a binary sequence specifying the set of + byte values to be removed. If omitted or ``None``, the *bytes* + argument defaults to removing ASCII whitespace. The *bytes* argument is not a prefix or suffix; rather, all combinations of its values are stripped:: @@ -3896,8 +4364,8 @@ place, and instead produce new objects. always produces a new object, even if no changes were made. -.. method:: bytes.zfill(width) - bytearray.zfill(width) +.. method:: bytes.zfill(width, /) + bytearray.zfill(width, /) Return a copy of the sequence left filled with ASCII ``b'0'`` digits to make a sequence of length *width*. A leading sign prefix (``b'+'``/ @@ -4145,7 +4613,10 @@ copying. types such as :class:`bytes` and :class:`bytearray`, an element is a single byte, but other types such as :class:`array.array` may have bigger elements. - ``len(view)`` is equal to the length of :class:`~memoryview.tolist`, which + :class:`!memoryview`\s are :ref:`generic ` over the type of their + underlying data. + + ``len(view)`` is equal to the length of :meth:`~memoryview.tolist`, which is the nested list representation of the view. If ``view.ndim = 1``, this is equal to the number of elements in the view. @@ -4309,7 +4780,8 @@ copying. in-memory Fortran order is preserved. For non-contiguous views, the data is converted to C first. *order=None* is the same as *order='C'*. - .. method:: hex([sep[, bytes_per_sep]]) + .. method:: hex(*, bytes_per_sep=1) + hex(sep, bytes_per_sep=1) Return a string object containing two hexadecimal digits for each byte in the buffer. :: @@ -4394,7 +4866,8 @@ copying. .. versionadded:: 3.2 - .. method:: cast(format[, shape]) + .. method:: cast(format, /) + cast(format, shape, /) Cast a memoryview to a new format or shape. *shape* defaults to ``[byte_length//new_itemsize]``, which means that the result view @@ -4493,7 +4966,7 @@ copying. .. versionadded:: 3.14 - .. method:: index(value, start=0, stop=sys.maxsize, /) + .. method:: index(value, start=0, stop=sys.maxsize, /) Return the index of the first occurrence of *value* (at or after index *start* and before index *stop*). @@ -4622,6 +5095,9 @@ copying. .. versionadded:: 3.3 +For information on the thread safety of :class:`memoryview` objects in +the :term:`free-threaded build`, see :ref:`thread-safety-memoryview`. + .. _types-set: @@ -4644,11 +5120,12 @@ other sequence-like behavior. There are currently two built-in set types, :class:`set` and :class:`frozenset`. The :class:`set` type is mutable --- the contents can be changed using methods -like :meth:`~set.add` and :meth:`~set.remove`. Since it is mutable, it has no -hash value and cannot be used as either a dictionary key or as an element of -another set. The :class:`frozenset` type is immutable and :term:`hashable` --- -its contents cannot be altered after it is created; it can therefore be used as -a dictionary key or as an element of another set. +like :meth:`~set.add` and :meth:`~set.remove`. +Since it is mutable, it has no hash value and cannot be used as +either a dictionary key or as an element of another set. +The :class:`frozenset` type is immutable and :term:`hashable` --- +its contents cannot be altered after it is created; +it can therefore be used as a dictionary key or as an element of another set. Non-empty sets (not frozensets) can be created by placing a comma-separated list of elements within braces, for example: ``{'jack', 'sjoerd'}``, in addition to the @@ -4656,8 +5133,8 @@ of elements within braces, for example: ``{'jack', 'sjoerd'}``, in addition to t The constructors for both classes work the same: -.. class:: set([iterable]) - frozenset([iterable]) +.. class:: set(iterable=(), /) + frozenset(iterable=(), /) Return a new set or frozenset object whose elements are taken from *iterable*. The elements of a set must be :term:`hashable`. To @@ -4665,170 +5142,185 @@ The constructors for both classes work the same: objects. If *iterable* is not specified, a new empty set is returned. - Sets can be created by several means: +Sets can be created by several means: - * Use a comma-separated list of elements within braces: ``{'jack', 'sjoerd'}`` - * Use a set comprehension: ``{c for c in 'abracadabra' if c not in 'abc'}`` - * Use the type constructor: ``set()``, ``set('foobar')``, ``set(['a', 'b', 'foo'])`` +* Use a comma-separated list of elements within braces: ``{'jack', 'sjoerd'}`` +* Use a set comprehension: ``{c for c in 'abracadabra' if c not in 'abc'}`` +* Use the type constructor: ``set()``, ``set('foobar')``, ``set(['a', 'b', 'foo'])`` - Instances of :class:`set` and :class:`frozenset` provide the following - operations: +Instances of :class:`set` and :class:`frozenset` provide the following +operations: - .. describe:: len(s) +.. describe:: len(s) - Return the number of elements in set *s* (cardinality of *s*). + Return the number of elements in set *s* (cardinality of *s*). - .. describe:: x in s +.. describe:: x in s - Test *x* for membership in *s*. + Test *x* for membership in *s*. - .. describe:: x not in s +.. describe:: x not in s - Test *x* for non-membership in *s*. + Test *x* for non-membership in *s*. - .. method:: isdisjoint(other) +.. method:: frozenset.isdisjoint(other, /) + set.isdisjoint(other, /) - Return ``True`` if the set has no elements in common with *other*. Sets are - disjoint if and only if their intersection is the empty set. + Return ``True`` if the set has no elements in common with *other*. Sets are + disjoint if and only if their intersection is the empty set. - .. method:: issubset(other) - set <= other +.. method:: frozenset.issubset(other, /) + set.issubset(other, /) +.. describe:: set <= other - Test whether every element in the set is in *other*. + Test whether every element in the set is in *other*. - .. method:: set < other +.. describe:: set < other - Test whether the set is a proper subset of *other*, that is, - ``set <= other and set != other``. + Test whether the set is a proper subset of *other*, that is, + ``set <= other and set != other``. - .. method:: issuperset(other) - set >= other +.. method:: frozenset.issuperset(other, /) + set.issuperset(other, /) +.. describe:: set >= other - Test whether every element in *other* is in the set. + Test whether every element in *other* is in the set. - .. method:: set > other +.. describe:: set > other - Test whether the set is a proper superset of *other*, that is, ``set >= - other and set != other``. + Test whether the set is a proper superset of *other*, that is, ``set >= + other and set != other``. - .. method:: union(*others) - set | other | ... +.. method:: frozenset.union(*others) + set.union(*others) +.. describe:: set | other | ... - Return a new set with elements from the set and all others. + Return a new set with elements from the set and all others. - .. method:: intersection(*others) - set & other & ... +.. method:: frozenset.intersection(*others) + set.intersection(*others) +.. describe:: set & other & ... - Return a new set with elements common to the set and all others. + Return a new set with elements common to the set and all others. - .. method:: difference(*others) - set - other - ... +.. method:: frozenset.difference(*others) + set.difference(*others) +.. describe:: set - other - ... - Return a new set with elements in the set that are not in the others. + Return a new set with elements in the set that are not in the others. - .. method:: symmetric_difference(other) - set ^ other +.. method:: frozenset.symmetric_difference(other, /) + set.symmetric_difference(other, /) +.. describe:: set ^ other - Return a new set with elements in either the set or *other* but not both. + Return a new set with elements in either the set or *other* but not both. - .. method:: copy() +.. method:: frozenset.copy() + set.copy() - Return a shallow copy of the set. + Return a shallow copy of the set. - Note, the non-operator versions of :meth:`union`, :meth:`intersection`, - :meth:`difference`, :meth:`symmetric_difference`, :meth:`issubset`, and - :meth:`issuperset` methods will accept any iterable as an argument. In - contrast, their operator based counterparts require their arguments to be - sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` - in favor of the more readable ``set('abc').intersection('cbs')``. +Note, the non-operator versions of :meth:`~frozenset.union`, +:meth:`~frozenset.intersection`, :meth:`~frozenset.difference`, :meth:`~frozenset.symmetric_difference`, :meth:`~frozenset.issubset`, and +:meth:`~frozenset.issuperset` methods will accept any iterable as an argument. In +contrast, their operator based counterparts require their arguments to be +sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` +in favor of the more readable ``set('abc').intersection('cbs')``. - Both :class:`set` and :class:`frozenset` support set to set comparisons. Two - sets are equal if and only if every element of each set is contained in the - other (each is a subset of the other). A set is less than another set if and - only if the first set is a proper subset of the second set (is a subset, but - is not equal). A set is greater than another set if and only if the first set - is a proper superset of the second set (is a superset, but is not equal). +Both :class:`set` and :class:`frozenset` support set to set comparisons. Two +sets are equal if and only if every element of each set is contained in the +other (each is a subset of the other). A set is less than another set if and +only if the first set is a proper subset of the second set (is a subset, but +is not equal). A set is greater than another set if and only if the first set +is a proper superset of the second set (is a superset, but is not equal). - Instances of :class:`set` are compared to instances of :class:`frozenset` - based on their members. For example, ``set('abc') == frozenset('abc')`` - returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. +Instances of :class:`set` are compared to instances of :class:`frozenset` +based on their members. For example, ``set('abc') == frozenset('abc')`` +returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. - The subset and equality comparisons do not generalize to a total ordering - function. For example, any two nonempty disjoint sets are not equal and are not - subsets of each other, so *all* of the following return ``False``: ``ab``. +The subset and equality comparisons do not generalize to a total ordering +function. For example, any two nonempty disjoint sets are not equal and are not +subsets of each other, so *all* of the following return ``False``: ``ab``. - Since sets only define partial ordering (subset relationships), the output of - the :meth:`list.sort` method is undefined for lists of sets. +Since sets only define partial ordering (subset relationships), the output of +the :meth:`list.sort` method is undefined for lists of sets. - Set elements, like dictionary keys, must be :term:`hashable`. +Set elements, like dictionary keys, must be :term:`hashable`. - Binary operations that mix :class:`set` instances with :class:`frozenset` - return the type of the first operand. For example: ``frozenset('ab') | - set('bc')`` returns an instance of :class:`frozenset`. +Binary operations that mix :class:`set` instances with :class:`frozenset` +return the type of the first operand. For example: ``frozenset('ab') | +set('bc')`` returns an instance of :class:`frozenset`. - The following table lists operations available for :class:`set` that do not - apply to immutable instances of :class:`frozenset`: +The following table lists operations available for :class:`set` that do not +apply to immutable instances of :class:`frozenset`: - .. method:: update(*others) - set |= other | ... +.. method:: set.update(*others) +.. describe:: set |= other | ... - Update the set, adding elements from all others. + Update the set, adding elements from all others. - .. method:: intersection_update(*others) - set &= other & ... +.. method:: set.intersection_update(*others) +.. describe:: set &= other & ... - Update the set, keeping only elements found in it and all others. + Update the set, keeping only elements found in it and all others. - .. method:: difference_update(*others) - set -= other | ... +.. method:: set.difference_update(*others) +.. describe:: set -= other | ... - Update the set, removing elements found in others. + Update the set, removing elements found in others. - .. method:: symmetric_difference_update(other) - set ^= other +.. method:: set.symmetric_difference_update(other, /) +.. describe:: set ^= other - Update the set, keeping only elements found in either set, but not in both. + Update the set, keeping only elements found in either set, but not in both. - .. method:: add(elem) +.. method:: set.add(elem, /) - Add element *elem* to the set. + Add element *elem* to the set. - .. method:: remove(elem) +.. method:: set.remove(elem, /) - Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is - not contained in the set. + Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is + not contained in the set. - .. method:: discard(elem) +.. method:: set.discard(elem, /) - Remove element *elem* from the set if it is present. + Remove element *elem* from the set if it is present. - .. method:: pop() +.. method:: set.pop() - Remove and return an arbitrary element from the set. Raises - :exc:`KeyError` if the set is empty. + Remove and return an arbitrary element from the set. Raises + :exc:`KeyError` if the set is empty. - .. method:: clear() +.. method:: set.clear() + + Remove all elements from the set. + + +Note, the non-operator versions of the :meth:`~set.update`, +:meth:`~set.intersection_update`, :meth:`~set.difference_update`, and +:meth:`~set.symmetric_difference_update` methods will accept any iterable as an +argument. - Remove all elements from the set. +Note, the *elem* argument to the :meth:`~object.__contains__`, +:meth:`~set.remove`, and +:meth:`~set.discard` methods may be a set. To support searching for an equivalent +frozenset, a temporary one is created from *elem*. +Sets and frozensets are :ref:`generic ` over the type of their elements. - Note, the non-operator versions of the :meth:`update`, - :meth:`intersection_update`, :meth:`difference_update`, and - :meth:`symmetric_difference_update` methods will accept any iterable as an - argument. +.. seealso:: - Note, the *elem* argument to the :meth:`~object.__contains__`, - :meth:`remove`, and - :meth:`discard` methods may be a set. To support searching for an equivalent - frozenset, a temporary one is created from *elem*. + For detailed information on thread-safety guarantees for :class:`set` + objects, see :ref:`thread-safety-set`. .. _typesmapping: -Mapping Types --- :class:`dict` -=============================== +Mapping types --- :class:`!dict`, :class:`!frozendict` +====================================================== .. index:: pair: object; mapping @@ -4839,8 +5331,9 @@ Mapping Types --- :class:`dict` pair: built-in function; len A :term:`mapping` object maps :term:`hashable` values to arbitrary objects. -Mappings are mutable objects. There is currently only one standard mapping -type, the :dfn:`dictionary`. (For other containers see the built-in +There are currently two standard mapping types, the :dfn:`dictionary` and +:class:`frozendict`. +(For other containers see the built-in :class:`list`, :class:`set`, and :class:`tuple` classes, and the :mod:`collections` module.) @@ -4852,8 +5345,8 @@ Values that compare equal (such as ``1``, ``1.0``, and ``True``) can be used interchangeably to index the same dictionary entry. .. class:: dict(**kwargs) - dict(mapping, **kwargs) - dict(iterable, **kwargs) + dict(mapping, /, **kwargs) + dict(iterable, /, **kwargs) Return a new dictionary initialized from an optional positional argument and a possibly empty set of keyword arguments. @@ -4881,9 +5374,6 @@ can be used interchangeably to index the same dictionary entry. being added is already present, the value from the keyword argument replaces the value from the positional argument. - Providing keyword arguments as in the first example only works for keys that - are valid Python identifiers. Otherwise, any valid keys can be used. - Dictionaries compare equal if and only if they have the same ``(key, value)`` pairs (regardless of ordering). Order comparisons ('<', '<=', '>=', '>') raise :exc:`TypeError`. To illustrate dictionary creation and equality, @@ -4924,6 +5414,9 @@ can be used interchangeably to index the same dictionary entry. Dictionary order is guaranteed to be insertion order. This behavior was an implementation detail of CPython from 3.6. + Dictionaries are :ref:`generic ` over two types, signifying + (respectively) the types of the dictionary's keys and values. + These are the operations that dictionaries support (and therefore, custom mapping types should support too): @@ -4942,13 +5435,13 @@ can be used interchangeably to index the same dictionary entry. .. index:: __missing__() - If a subclass of dict defines a method :meth:`__missing__` and *key* + If a subclass of dict defines a method :meth:`~object.__missing__` and *key* is not present, the ``d[key]`` operation calls that method with the key *key* as argument. The ``d[key]`` operation then returns or raises whatever is returned or raised by the ``__missing__(key)`` call. - No other operations or methods invoke :meth:`__missing__`. If - :meth:`__missing__` is not defined, :exc:`KeyError` is raised. - :meth:`__missing__` must be a method; it cannot be an instance variable:: + No other operations or methods invoke :meth:`~object.__missing__`. If + :meth:`~object.__missing__` is not defined, :exc:`KeyError` is raised. + :meth:`~object.__missing__` must be a method; it cannot be an instance variable:: >>> class Counter(dict): ... def __missing__(self, key): @@ -4962,7 +5455,8 @@ can be used interchangeably to index the same dictionary entry. 1 The example above shows part of the implementation of - :class:`collections.Counter`. A different ``__missing__`` method is used + :class:`collections.Counter`. + A different :meth:`!__missing__` method is used by :class:`collections.defaultdict`. .. describe:: d[key] = value @@ -5021,7 +5515,8 @@ can be used interchangeably to index the same dictionary entry. Return a new view of the dictionary's keys. See the :ref:`documentation of view objects `. - .. method:: pop(key[, default]) + .. method:: pop(key, /) + pop(key, default, /) If *key* is in the dictionary, remove it and return its value, else return *default*. If *default* is not given and *key* is not in the dictionary, @@ -5053,9 +5548,11 @@ can be used interchangeably to index the same dictionary entry. with a value of *default* and return *default*. *default* defaults to ``None``. - .. method:: update([other]) + .. method:: update(**kwargs) + update(mapping, /, **kwargs) + update(iterable, /, **kwargs) - Update the dictionary with the key/value pairs from *other*, overwriting + Update the dictionary with the key/value pairs from *mapping* or *iterable* and *kwargs*, overwriting existing keys. Return ``None``. :meth:`update` accepts either another object with a ``keys()`` method (in @@ -5109,9 +5606,14 @@ can be used interchangeably to index the same dictionary entry. Dictionaries are now reversible. + .. seealso:: + :class:`frozendict` and :class:`types.MappingProxyType` can be used to + create a read-only view of a :class:`dict`. + .. seealso:: - :class:`types.MappingProxyType` can be used to create a read-only view - of a :class:`dict`. + + For detailed information on thread-safety guarantees for :class:`dict` + objects, see :ref:`thread-safety-dict`. .. _dict-views: @@ -5220,6 +5722,44 @@ An example of dictionary view usage:: 500 +Frozen dictionaries +------------------- + +.. class:: frozendict(**kwargs) + frozendict(mapping, /, **kwargs) + frozendict(iterable, /, **kwargs) + + Return a new frozen dictionary initialized from an optional positional + argument and a possibly empty set of keyword arguments. + + A :class:`!frozendict` has a similar API to the :class:`dict` API, with the + following differences: + + * :class:`!dict` has more methods than :class:`!frozendict`: + + * :meth:`!__delitem__` + * :meth:`!__setitem__` + * :meth:`~dict.clear` + * :meth:`~dict.pop` + * :meth:`~dict.popitem` + * :meth:`~dict.setdefault` + * :meth:`~dict.update` + + * A :class:`!frozendict` can be hashed with ``hash(frozendict)`` if all keys and + values can be hashed. + + * ``frozendict |= other`` does not modify the :class:`!frozendict` in-place but + creates a new frozen dictionary. + + :class:`!frozendict` is not a :class:`!dict` subclass but inherits directly + from ``object``. + + Like dictionaries, frozendicts are :ref:`generic ` over two types, + signifying (respectively) the types of the frozendict's keys and values. + + .. versionadded:: 3.15 + + .. _typecontextmanager: Context Manager Types @@ -5265,9 +5805,11 @@ before the statement body is executed and exited when the statement ends: Returning a true value from this method will cause the :keyword:`with` statement to suppress the exception and continue execution with the statement immediately following the :keyword:`!with` statement. Otherwise the exception continues - propagating after this method has finished executing. Exceptions that occur - during execution of this method will replace any exception that occurred in the - body of the :keyword:`!with` statement. + propagating after this method has finished executing. + + If this method raises an exception while handling an earlier exception from the + :keyword:`with` block, the new exception is raised, and the original exception + is stored in its :attr:`~BaseException.__context__` attribute. The exception passed in should never be reraised explicitly - instead, this method should return a false value to indicate that the method completed @@ -5356,7 +5898,8 @@ type and the :class:`bytes` data type: ``GenericAlias`` objects are instances of the class :class:`types.GenericAlias`, which can also be used to create ``GenericAlias`` -objects directly. +objects directly. Specializations of user-defined :ref:`generic classes ` +may not be instances of :class:`types.GenericAlias`, but they provide similar functionality. .. describe:: T[X, Y, ...] @@ -5405,6 +5948,15 @@ creation:: >>> type(l) + +Instances of ``GenericAlias`` are not classes at runtime, even though they behave like classes (they can be instantiated and subclassed):: + + >>> import inspect + >>> inspect.isclass(list[int]) + False + +This is true for :ref:`user-defined generics ` also. + Calling :func:`repr` or :func:`str` on a generic shows the parameterized type:: >>> repr(list[int]) @@ -5441,6 +5993,7 @@ list is non-exhaustive. * :class:`list` * :class:`dict` * :class:`set` +* :class:`frozendict` * :class:`frozenset` * :class:`type` * :class:`asyncio.Future` @@ -5468,6 +6021,7 @@ list is non-exhaustive. * :class:`collections.abc.MutableMapping` * :class:`collections.abc.Sequence` * :class:`collections.abc.MutableSequence` +* :class:`collections.abc.ByteString` * :class:`collections.abc.MappingView` * :class:`collections.abc.KeysView` * :class:`collections.abc.ItemsView` @@ -5743,9 +6297,10 @@ Methods .. index:: pair: object; method -Methods are functions that are called using the attribute notation. There are -two flavors: :ref:`built-in methods ` (such as :meth:`append` -on lists) and :ref:`class instance method `. +Methods are functions that are called using the attribute notation. +There are two flavors: :ref:`built-in methods ` +(such as :meth:`~list.append` on lists) +and :ref:`class instance method `. Built-in methods are described with the types that support them. If you access a method (a function defined in a class namespace) through an @@ -5851,13 +6406,34 @@ It is written as ``None``. The Ellipsis Object ------------------- -This object is commonly used by slicing (see :ref:`slicings`). It supports no -special operations. There is exactly one ellipsis object, named +This object is commonly used to indicate that something is omitted. +It supports no special operations. There is exactly one ellipsis object, named :const:`Ellipsis` (a built-in name). ``type(Ellipsis)()`` produces the :const:`Ellipsis` singleton. It is written as ``Ellipsis`` or ``...``. +In typical use, ``...`` as the ``Ellipsis`` object appears in a few different +places, for instance: + +- In type annotations, such as :ref:`callable arguments ` + or :ref:`tuple elements `. + +- As the body of a function instead of a :ref:`pass statement `. + +- In third-party libraries, such as `Numpy's slicing and striding + `_. + +Python also uses three dots in ways that are not ``Ellipsis`` objects, for instance: + +- Doctest's :const:`ELLIPSIS `, as a pattern for missing content. + +- The default Python prompt of the :term:`interactive` shell when partial input is incomplete. + +Lastly, the Python documentation often uses three dots in conventional English +usage to mean omitted content, even in code examples that also use them as the +``Ellipsis``. + .. _bltin-notimplemented-object: diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 83e8ee2722ed8ae..be968a3c53d8430 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -4,7 +4,7 @@ .. module:: string :synopsis: Common string operations. -**Source code:** :source:`Lib/string.py` +**Source code:** :source:`Lib/string/__init__.py` -------------- @@ -82,12 +82,12 @@ The constants defined in this module are: .. _string-formatting: -Custom String Formatting +Custom string formatting ------------------------ The built-in string class provides the ability to do complex variable substitutions and value formatting via the :meth:`~str.format` method described in -:pep:`3101`. The :class:`Formatter` class in the :mod:`string` module allows +:pep:`3101`. The :class:`Formatter` class in the :mod:`!string` module allows you to create and customize your own string formatting behaviors using the same implementation as the built-in :meth:`~str.format` method. @@ -192,7 +192,7 @@ implementation as the built-in :meth:`~str.format` method. .. _formatstrings: -Format String Syntax +Format string syntax -------------------- The :meth:`str.format` method and the :class:`Formatter` class share the same @@ -200,7 +200,7 @@ syntax for format strings (although in the case of :class:`Formatter`, subclasses can define their own format string syntax). The syntax is related to that of :ref:`formatted string literals ` and :ref:`template string literals `, but it is less sophisticated -and, in particular, does not support arbitrary expressions. +and, in particular, does not support arbitrary expressions in interpolations. .. index:: single: {} (curly brackets); in string formatting @@ -304,7 +304,7 @@ See the :ref:`formatexamples` section for some examples. .. _formatspec: -Format Specification Mini-Language +Format specification mini-language ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ "Format specifications" are used within replacement fields contained within a @@ -472,7 +472,9 @@ of a number respectively. It can be one of the following: | | this option is not supported. | +---------+----------------------------------------------------------+ -For a locale aware separator, use the ``'n'`` presentation type instead. +For a locale-aware separator, use the ``'n'`` +:ref:`float presentation type ` or +:ref:`integer presentation type ` instead. .. versionchanged:: 3.1 Added the ``','`` option (see also :pep:`378`). @@ -518,9 +520,14 @@ The available integer presentation types are: | | In case ``'#'`` is specified, the prefix ``'0x'`` will | | | be upper-cased to ``'0X'`` as well. | +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'d'``, except that it uses | + | ``'n'`` | .. _n-format-integer: | + | | | + | | Number. This is the same as ``'d'``, except that it uses | | | the current locale setting to insert the appropriate | - | | digit group separators. | + | | digit group separators. Note that the default locale is | + | | not the system locale. Depending on your use case, you | + | | may wish to set :const:`~locale.LC_NUMERIC` with | + | | :func:`locale.setlocale` before using ``'n'``. | +---------+----------------------------------------------------------+ | None | The same as ``'d'``. | +---------+----------------------------------------------------------+ @@ -546,6 +553,9 @@ The available presentation types for :class:`float` and | | :class:`float`, and shows all coefficient digits | | | for :class:`~decimal.Decimal`. If ``p=0``, the decimal | | | point is omitted unless the ``#`` option is used. | + | | | + | | For :class:`float`, the exponent always contains at | + | | least two digits, and is zero if the value is zero. | +---------+----------------------------------------------------------+ | ``'E'`` | Scientific notation. Same as ``'e'`` except it uses | | | an upper case 'E' as the separator character. | @@ -600,10 +610,15 @@ The available presentation types for :class:`float` and | | ``'E'`` if the number gets too large. The | | | representations of infinity and NaN are uppercased, too. | +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'g'``, except that it uses | + | ``'n'`` | .. _n-format-float: | + | | | + | | Number. This is the same as ``'g'``, except that it uses | | | the current locale setting to insert the appropriate | - | | digit group separators | - | | for the integral part of a number. | + | | digit group separators for the integral part of a | + | | number. Note that the default locale is not the system | + | | locale. Depending on your use case, you may wish to set | + | | :const:`~locale.LC_NUMERIC` with | + | | :func:`locale.setlocale` before using ``'n'``. | +---------+----------------------------------------------------------+ | ``'%'`` | Percentage. Multiplies the number by 100 and displays | | | in fixed (``'f'``) format, followed by a percent sign. | @@ -756,8 +771,8 @@ Expressing a percentage:: Using type-specific formatting:: - >>> import datetime - >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58) + >>> import datetime as dt + >>> d = dt.datetime(2010, 7, 4, 12, 15, 58) >>> '{:%Y-%m-%d %H:%M:%S}'.format(d) '2010-07-04 12:15:58' @@ -799,13 +814,15 @@ Template strings ($-strings) .. note:: - The feature described here was introduced in Python 2.4. It is unrelated - to, and should not be confused with, the newer - :ref:`template strings ` and - :ref:`t-string literal syntax ` introduced in Python 3.14. - T-string literals evaluate to instances of a different - :class:`~string.templatelib.Template` class, found in the - :mod:`string.templatelib` module. + The feature described here was introduced in Python 2.4; + a simple templating method based upon regular expressions. + It predates :meth:`str.format`, :ref:`formatted string literals `, + and :ref:`template string literals `. + + It is unrelated to template string literals (t-strings), + which were introduced in Python 3.14. + These evaluate to :class:`string.templatelib.Template` objects, + found in the :mod:`string.templatelib` module. Template strings provide simpler string substitutions as described in :pep:`292`. A primary use case for template strings is for @@ -835,7 +852,7 @@ Template strings support ``$``-based substitutions, using the following rules: Any other appearance of ``$`` in the string will result in a :exc:`ValueError` being raised. -The :mod:`string` module provides a :class:`Template` class that implements +The :mod:`!string` module provides a :class:`Template` class that implements these rules. The methods of :class:`Template` are: diff --git a/Doc/library/string.templatelib.rst b/Doc/library/string.templatelib.rst index 31b90d75f411f0f..6e91850fdf59ca4 100644 --- a/Doc/library/string.templatelib.rst +++ b/Doc/library/string.templatelib.rst @@ -11,8 +11,8 @@ .. seealso:: * :ref:`Format strings ` - * :ref:`T-string literal syntax ` - + * :ref:`Template string literal (t-string) syntax ` + * :pep:`750` .. _template-strings: @@ -21,278 +21,316 @@ Template strings .. versionadded:: 3.14 -Template strings are a formatting mechanism that allows for deep control over -how strings are processed. You can create templates using -:ref:`t-string literal syntax `, which is identical to -:ref:`f-string syntax ` but uses a ``t`` instead of an ``f``. -While f-strings evaluate to ``str``, t-strings create a :class:`Template` -instance that gives you access to the static and interpolated (in curly braces) -parts of a string *before* they are combined. - - -.. _templatelib-template: +Template strings are a mechanism for custom string processing. +They have the full flexibility of Python's :ref:`f-strings`, +but return a :class:`Template` instance that gives access +to the static and interpolated (in curly brackets) parts of a string +*before* they are combined. -Template --------- +To write a t-string, use a ``'t'`` prefix instead of an ``'f'``, like so: -The :class:`!Template` class describes the contents of a template string. +.. code-block:: pycon -:class:`!Template` instances are immutable: their attributes cannot be -reassigned. + >>> pi = 3.14 + >>> t't-strings are new in Python {pi!s}!' + Template( + strings=('t-strings are new in Python ', '!'), + interpolations=(Interpolation(3.14, 'pi', 's', ''),) + ) -.. class:: Template(*args) +Types +----- - Create a new :class:`!Template` object. +.. class:: Template - :param args: A mix of strings and :class:`Interpolation` instances in any order. - :type args: str | Interpolation + The :class:`!Template` class describes the contents of a template string. + It is immutable, meaning that attributes of a template cannot be reassigned. The most common way to create a :class:`!Template` instance is to use the - :ref:`t-string literal syntax `. This syntax is identical to that of - :ref:`f-strings ` except that it uses a ``t`` instead of an ``f``: + :ref:`template string literal syntax `. + This syntax is identical to that of :ref:`f-strings `, + except that it uses a ``t`` prefix in place of an ``f``: - >>> name = "World" - >>> template = t"Hello {name}!" + >>> cheese = 'Red Leicester' + >>> template = t"We're fresh out of {cheese}, sir." >>> type(template) - Templates ars stored as sequences of literal :attr:`~Template.strings` + Templates are stored as sequences of literal :attr:`~Template.strings` and dynamic :attr:`~Template.interpolations`. - A :attr:`~Template.values` attribute holds the interpolation values: + A :attr:`~Template.values` attribute holds the values of the interpolations: + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' >>> template.strings - ('Hello ', '!') + ('Ah! We do have ', '.') >>> template.interpolations - (Interpolation('World', ...),) + (Interpolation('Camembert', ...),) >>> template.values - ('World',) + ('Camembert',) The :attr:`!strings` tuple has one more element than :attr:`!interpolations` and :attr:`!values`; the interpolations “belong” between the strings. - This may be easier to understand when tuples are aligned:: + This may be easier to understand when tuples are aligned - template.strings: ('Hello ', '!') - template.values: ( 'World', ) + .. code-block:: python - While literal syntax is the most common way to create :class:`!Template` - instances, it is also possible to create them directly using the constructor: + template.strings: ('Ah! We do have ', '.') + template.values: ( 'Camembert', ) - >>> from string.templatelib import Interpolation, Template - >>> name = "World" - >>> template = Template("Hello, ", Interpolation(name, "name"), "!") - >>> list(template) - ['Hello, ', Interpolation('World', 'name', None, ''), '!'] + .. rubric:: Attributes - If two or more consecutive strings are passed, they will be concatenated - into a single value in the :attr:`~Template.strings` attribute. For example, - the following code creates a :class:`Template` with a single final string: + .. attribute:: strings + :type: tuple[str, ...] - >>> from string.templatelib import Template - >>> template = Template("Hello ", "World", "!") - >>> template.strings - ('Hello World!',) + A :class:`tuple` of the static strings in the template. - If two or more consecutive interpolations are passed, they will be treated - as separate interpolations and an empty string will be inserted between them. - For example, the following code creates a template with empty placeholders - in the :attr:`~Template.strings` attribute: + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' + >>> template.strings + ('Ah! We do have ', '.') - >>> from string.templatelib import Interpolation, Template - >>> template = Template(Interpolation("World", "name"), Interpolation("!", "punctuation")) - >>> template.strings - ('', '', '') + Empty strings *are* included in the tuple: - .. attribute:: strings - :type: tuple[str, ...] + >>> response = 'We do have ' + >>> cheese = 'Camembert' + >>> template = t'Ah! {response}{cheese}.' + >>> template.strings + ('Ah! ', '', '.') - A :ref:`tuple ` of the static strings in the template. + The ``strings`` tuple is never empty, and always contains one more + string than the ``interpolations`` and ``values`` tuples: - >>> name = "World" - >>> t"Hello {name}!".strings - ('Hello ', '!') + >>> t''.strings + ('',) + >>> t''.values + () + >>> t'{'cheese'}'.strings + ('', '') + >>> t'{'cheese'}'.values + ('cheese',) - Empty strings *are* included in the tuple: + .. attribute:: interpolations + :type: tuple[Interpolation, ...] - >>> name = "World" - >>> t"Hello {name}{name}!".strings - ('Hello ', '', '!') + A :class:`tuple` of the interpolations in the template. - The ``strings`` tuple is never empty, and always contains one more - string than the ``interpolations`` and ``values`` tuples: + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' + >>> template.interpolations + (Interpolation('Camembert', 'cheese', None, ''),) - >>> t"".strings - ('',) - >>> t"".values - () - >>> t"{'cheese'}".strings - ('', '') - >>> t"{'cheese'}".values - ('cheese',) + The ``interpolations`` tuple may be empty and always contains one fewer + values than the ``strings`` tuple: - .. attribute:: interpolations - :type: tuple[Interpolation, ...] + >>> t'Red Leicester'.interpolations + () - A tuple of the interpolations in the template. + .. attribute:: values + :type: tuple[object, ...] - >>> name = "World" - >>> t"Hello {name}!".interpolations - (Interpolation('World', 'name', None, ''),) + A tuple of all interpolated values in the template. - The ``interpolations`` tuple may be empty and always contains one fewer - values than the ``strings`` tuple: + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' + >>> template.values + ('Camembert',) - >>> t"Hello!".interpolations - () + The ``values`` tuple always has the same length as the + ``interpolations`` tuple. It is always equivalent to + ``tuple(i.value for i in template.interpolations)``. - .. attribute:: values - :type: tuple[Any, ...] + .. rubric:: Methods - A tuple of all interpolated values in the template. + .. method:: __new__(*args: str | Interpolation) - >>> name = "World" - >>> t"Hello {name}!".values - ('World',) + While literal syntax is the most common way to create a :class:`!Template`, + it is also possible to create them directly using the constructor: - The ``values`` tuple always has the same length as the - ``interpolations`` tuple. It is equivalent to - ``tuple(i.value for i in template.interpolations)``. + >>> from string.templatelib import Interpolation, Template + >>> cheese = 'Camembert' + >>> template = Template( + ... 'Ah! We do have ', Interpolation(cheese, 'cheese'), '.' + ... ) + >>> list(template) + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, ''), '.'] - .. describe:: iter(template) + If multiple strings are passed consecutively, they will be concatenated + into a single value in the :attr:`~Template.strings` attribute. For example, + the following code creates a :class:`Template` with a single final string: - Iterate over the template, yielding each string and - :class:`Interpolation` in order. + >>> from string.templatelib import Template + >>> template = Template('Ah! We do have ', 'Camembert', '.') + >>> template.strings + ('Ah! We do have Camembert.',) - >>> name = "World" - >>> list(t"Hello {name}!") - ['Hello ', Interpolation('World', 'name', None, ''), '!'] + If multiple interpolations are passed consecutively, they will be treated + as separate interpolations and an empty string will be inserted between them. + For example, the following code creates a template with empty placeholders + in the :attr:`~Template.strings` attribute: - Empty strings are *not* included in the iteration: + >>> from string.templatelib import Interpolation, Template + >>> template = Template( + ... Interpolation('Camembert', 'cheese'), + ... Interpolation('.', 'punctuation'), + ... ) + >>> template.strings + ('', '', '') - >>> name = "World" - >>> list(t"Hello {name}{name}") - ['Hello ', Interpolation('World', 'name', None, ''), Interpolation('World', 'name', None, '')] + .. describe:: iter(template) - .. describe:: template + other - template += other + Iterate over the template, yielding each non-empty string and + :class:`Interpolation` in the correct order: - Concatenate this template with another, returning a new - :class:`!Template` instance: + >>> cheese = 'Camembert' + >>> list(t'Ah! We do have {cheese}.') + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, ''), '.'] - >>> name = "World" - >>> list(t"Hello " + t"there {name}!") - ['Hello there ', Interpolation('World', 'name', None, ''), '!'] + .. caution:: - Concatenation between a :class:`!Template` and a ``str`` is *not* supported. - This is because it is ambiguous whether the string should be treated as - a static string or an interpolation. If you want to concatenate a - :class:`!Template` with a string, you should either wrap the string - directly in a :class:`!Template` (to treat it as a static string) or use - an :class:`!Interpolation` (to treat it as dynamic): + Empty strings are **not** included in the iteration: - >>> from string.templatelib import Template, Interpolation - >>> template = t"Hello " - >>> # Treat "there " as a static string - >>> template += Template("there ") - >>> # Treat name as an interpolation - >>> name = "World" - >>> template += Template(Interpolation(name, "name")) - >>> list(template) - ['Hello there ', Interpolation('World', 'name', None, '')] + >>> response = 'We do have ' + >>> cheese = 'Camembert' + >>> list(t'Ah! {response}{cheese}.') # doctest: +NORMALIZE_WHITESPACE + ['Ah! ', + Interpolation('We do have ', 'response', None, ''), + Interpolation('Camembert', 'cheese', None, ''), + '.'] + .. describe:: template + other + template += other -.. class:: Interpolation(value, expression="", conversion=None, format_spec="") + Concatenate this template with another, returning a new + :class:`!Template` instance: - Create a new :class:`!Interpolation` object. + >>> cheese = 'Camembert' + >>> list(t'Ah! ' + t'We do have {cheese}.') + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, ''), '.'] - :param value: The evaluated, in-scope result of the interpolation. - :type value: object + Concatenating a :class:`!Template` and a ``str`` is **not** supported. + This is because it is unclear whether the string should be treated as + a static string or an interpolation. + If you want to concatenate a :class:`!Template` with a string, + you should either wrap the string directly in a :class:`!Template` + (to treat it as a static string) + or use an :class:`!Interpolation` (to treat it as dynamic): - :param expression: The text of a valid Python expression, or an empty string. - :type expression: str + >>> from string.templatelib import Interpolation, Template + >>> template = t'Ah! ' + >>> # Treat 'We do have ' as a static string + >>> template += Template('We do have ') + >>> # Treat cheese as an interpolation + >>> cheese = 'Camembert' + >>> template += Template(Interpolation(cheese, 'cheese')) + >>> list(template) + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, '')] - :param conversion: The optional :ref:`conversion ` to be used, one of r, s, and a. - :type conversion: ``Literal["a", "r", "s"] | None`` - :param format_spec: An optional, arbitrary string used as the :ref:`format specification ` to present the value. - :type format_spec: str +.. class:: Interpolation The :class:`!Interpolation` type represents an expression inside a template string. + It is immutable, meaning that attributes of an interpolation cannot be reassigned. - :class:`!Interpolation` instances are immutable: their attributes cannot be - reassigned. + Interpolations support pattern matching, allowing you to match against + their attributes with the :ref:`match statement `: + + >>> from string.templatelib import Interpolation + >>> interpolation = t'{1. + 2.:.2f}'.interpolations[0] + >>> interpolation + Interpolation(3.0, '1. + 2.', None, '.2f') + >>> match interpolation: + ... case Interpolation(value, expression, conversion, format_spec): + ... print(value, expression, conversion, format_spec, sep=' | ') + ... + 3.0 | 1. + 2. | None | .2f + + Interpolations are :ref:`generic ` over the types of their values. + + .. rubric:: Attributes .. attribute:: value + :type: object - :returns: The evaluated value of the interpolation. - :type: object + The evaluated value of the interpolation. - >>> t"{1 + 2}".interpolations[0].value - 3 + >>> t'{1 + 2}'.interpolations[0].value + 3 .. attribute:: expression + :type: str - :returns: The text of a valid Python expression, or an empty string. - :type: str + For interpolations created by t-string literals, :attr:`!expression` + is the expression text found inside the curly brackets (``{`` & ``}``), + including any whitespace, excluding the curly brackets themselves, + and ending before the first ``!``, ``:``, or ``=`` if any is present. + For manually created interpolations, :attr:`!expression` is the arbitrary + string provided when constructing the interpolation instance. - The :attr:`~Interpolation.expression` is the original text of the - interpolation's Python expression, if the interpolation was created - from a t-string literal. Developers creating interpolations manually - should either set this to an empty string or choose a suitable valid - Python expression. + We recommend using valid Python expressions or the empty string for the + ``expression`` field of manually created :class:`!Interpolation` + instances, although this is not enforced at runtime. - >>> t"{1 + 2}".interpolations[0].expression - '1 + 2' + >>> t'{1 + 2}'.interpolations[0].expression + '1 + 2' .. attribute:: conversion + :type: typing.Literal['a', 'r', 's'] | None - :returns: The conversion to apply to the value, or ``None``. - :type: ``Literal["a", "r", "s"] | None`` + The conversion to apply to the value, or ``None``. - The :attr:`!Interpolation.conversion` is the optional conversion to apply - to the value: + The :attr:`!conversion` is the optional conversion to apply + to the value: - >>> t"{1 + 2!a}".interpolations[0].conversion - 'a' + >>> t'{1 + 2!a}'.interpolations[0].conversion + 'a' - .. note:: + .. note:: Unlike f-strings, where conversions are applied automatically, the expected behavior with t-strings is that code that *processes* the :class:`!Template` will decide how to interpret and whether to apply - the :attr:`!Interpolation.conversion`. + the :attr:`!conversion`. + For convenience, the :func:`convert` function can be used to mimic + f-string conversion semantics. .. attribute:: format_spec + :type: str - :returns: The format specification to apply to the value. - :type: str + The format specification to apply to the value. - The :attr:`!Interpolation.format_spec` is an optional, arbitrary string - used as the format specification to present the value: + The :attr:`!format_spec` is an optional, arbitrary string + used as the format specification to present the value: - >>> t"{1 + 2:.2f}".interpolations[0].format_spec - '.2f' + >>> t'{1 + 2:.2f}'.interpolations[0].format_spec + '.2f' - .. note:: + .. note:: Unlike f-strings, where format specifications are applied automatically via the :func:`format` protocol, the expected behavior with - t-strings is that code that *processes* the :class:`!Template` will + t-strings is that code that *processes* the interpolation will decide how to interpret and whether to apply the format specification. - As a result, :attr:`!Interpolation.format_spec` values in - :class:`!Template` instances can be arbitrary strings, even those that - do not necessarily conform to the rules of Python's :func:`format` - protocol. + As a result, :attr:`!format_spec` values in interpolations + can be arbitrary strings, + including those that do not conform to the :func:`format` protocol. - Interpolations support pattern matching, allowing you to match against - their attributes with the :ref:`match statement `: + .. rubric:: Methods - >>> from string.templatelib import Interpolation - >>> interpolation = Interpolation(3.0, "1 + 2", None, ".2f") - >>> match interpolation: - ... case Interpolation(value, expression, conversion, format_spec): - ... print(value, expression, conversion, format_spec) - ... - 3.0 1 + 2 None .2f + .. method:: __new__(value: object, \ + expression: str, \ + conversion: typing.Literal['a', 'r', 's'] | None = None, \ + format_spec: str = '') + + Create a new :class:`!Interpolation` object from component parts. + + :param value: The evaluated, in-scope result of the interpolation. + :param expression: The text of a valid Python expression, + or an empty string. + :param conversion: The :ref:`conversion ` to be used, + one of ``None``, ``'a'``, ``'r'``, or ``'s'``. + :param format_spec: An optional, arbitrary string used as the + :ref:`format specification ` to present the value. Helper functions @@ -306,8 +344,8 @@ Helper functions Three conversion flags are currently supported: - * ``'s'`` which calls :func:`str` on the value, - * ``'r'`` which calls :func:`repr`, and - * ``'a'`` which calls :func:`ascii`. + * ``'s'`` which calls :func:`str` on the value (like ``!s``), + * ``'r'`` which calls :func:`repr` (like ``!r``), and + * ``'a'`` which calls :func:`ascii` (like ``!a``). If the conversion flag is ``None``, *obj* is returned unchanged. diff --git a/Doc/library/stringprep.rst b/Doc/library/stringprep.rst index 37d5adf0fa95411..325ac9ae7c5293a 100644 --- a/Doc/library/stringprep.rst +++ b/Doc/library/stringprep.rst @@ -4,9 +4,6 @@ .. module:: stringprep :synopsis: String preparation, as per RFC 3453 -.. moduleauthor:: Martin v. Löwis -.. sectionauthor:: Martin v. Löwis - **Source code:** :source:`Lib/stringprep.py` -------------- @@ -26,14 +23,14 @@ define which tables it uses, and what other optional parts of the ``stringprep`` procedure are part of the profile. One example of a ``stringprep`` profile is ``nameprep``, which is used for internationalized domain names. -The module :mod:`stringprep` only exposes the tables from :rfc:`3454`. As these +The module :mod:`!stringprep` only exposes the tables from :rfc:`3454`. As these tables would be very large to represent as dictionaries or lists, the module uses the Unicode character database internally. The module source code itself was generated using the ``mkstringprep.py`` utility. As a result, these tables are exposed as functions, not as data structures. There are two kinds of tables in the RFC: sets and mappings. For a set, -:mod:`stringprep` provides the "characteristic function", i.e. a function that +:mod:`!stringprep` provides the "characteristic function", i.e. a function that returns ``True`` if the parameter is part of the set. For mappings, it provides the mapping function: given the key, it returns the associated value. Below is a list of all functions available in the module. diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 17fc479fd0c8c93..ec872fddee05c70 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -36,7 +36,7 @@ and the C layer. responsible for defining byte ordering and padding between elements. See :ref:`struct-alignment` for details. -Several :mod:`struct` functions (and methods of :class:`Struct`) take a *buffer* +Several :mod:`!struct` functions (and methods of :class:`Struct`) take a *buffer* argument. This refers to objects that implement the :ref:`bufferobjects` and provide either a readable or read-writable buffer. The most common types used for that purpose are :class:`bytes` and :class:`bytearray`, but many other types @@ -111,7 +111,7 @@ Format Strings -------------- Format strings describe the data layout when -packing and unpacking data. They are built up from :ref:`format characters`, +packing and unpacking data. They are built up from :ref:`type codes `, which specify the type of data being packed/unpacked. In addition, special characters control the :ref:`byte order, size and alignment`. Each format string consists of an optional prefix character which @@ -183,8 +183,8 @@ Use :data:`sys.byteorder` to check the endianness of your system. Native size and alignment are determined using the C compiler's ``sizeof`` expression. This is always combined with native byte order. -Standard size depends only on the format character; see the table in -the :ref:`format-characters` section. +Standard size depends only on the type code; see the table in +the :ref:`type-codes` section. Note the difference between ``'@'`` and ``'='``: both use native byte order, but the size and alignment of the latter is standardized. @@ -208,12 +208,13 @@ Notes: count of zero. See :ref:`struct-examples`. +.. _type-codes: .. _format-characters: -Format Characters -^^^^^^^^^^^^^^^^^ +Type Codes +^^^^^^^^^^ -Format characters have the following meaning; the conversion between C and +Type codes (or format codes) have the following meaning; the conversion between C and Python values should be obvious given their types. The 'Standard size' column refers to the size of the packed value in bytes when using standard size; that is, when the format string starts with one of ``'<'``, ``'>'``, ``'!'`` or @@ -227,34 +228,34 @@ platform-dependent. +--------+--------------------------+--------------------+----------------+------------+ | ``c`` | :c:expr:`char` | bytes of length 1 | 1 | | +--------+--------------------------+--------------------+----------------+------------+ -| ``b`` | :c:expr:`signed char` | integer | 1 | \(1), \(2) | +| ``b`` | :c:expr:`signed char` | int | 1 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``B`` | :c:expr:`unsigned char` | integer | 1 | \(2) | +| ``B`` | :c:expr:`unsigned char` | int | 1 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ | ``?`` | :c:expr:`_Bool` | bool | 1 | \(1) | +--------+--------------------------+--------------------+----------------+------------+ -| ``h`` | :c:expr:`short` | integer | 2 | \(2) | +| ``h`` | :c:expr:`short` | int | 2 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``H`` | :c:expr:`unsigned short` | integer | 2 | \(2) | +| ``H`` | :c:expr:`unsigned short` | int | 2 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``i`` | :c:expr:`int` | integer | 4 | \(2) | +| ``i`` | :c:expr:`int` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``I`` | :c:expr:`unsigned int` | integer | 4 | \(2) | +| ``I`` | :c:expr:`unsigned int` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``l`` | :c:expr:`long` | integer | 4 | \(2) | +| ``l`` | :c:expr:`long` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``L`` | :c:expr:`unsigned long` | integer | 4 | \(2) | +| ``L`` | :c:expr:`unsigned long` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``q`` | :c:expr:`long long` | integer | 8 | \(2) | +| ``q`` | :c:expr:`long long` | int | 8 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``Q`` | :c:expr:`unsigned long | integer | 8 | \(2) | +| ``Q`` | :c:expr:`unsigned long | int | 8 | \(2) | | | long` | | | | +--------+--------------------------+--------------------+----------------+------------+ -| ``n`` | :c:type:`ssize_t` | integer | | \(3) | +| ``n`` | :c:type:`ssize_t` | int | | \(2), \(3) | +--------+--------------------------+--------------------+----------------+------------+ -| ``N`` | :c:type:`size_t` | integer | | \(3) | +| ``N`` | :c:type:`size_t` | int | | \(2), \(3) | +--------+--------------------------+--------------------+----------------+------------+ -| ``e`` | \(6) | float | 2 | \(4) | +| ``e`` | :c:expr:`_Float16` | float | 2 | \(4), \(6) | +--------+--------------------------+--------------------+----------------+------------+ | ``f`` | :c:expr:`float` | float | 4 | \(4) | +--------+--------------------------+--------------------+----------------+------------+ @@ -264,11 +265,15 @@ platform-dependent. +--------+--------------------------+--------------------+----------------+------------+ | ``D`` | :c:expr:`double complex` | complex | 16 | \(10) | +--------+--------------------------+--------------------+----------------+------------+ +| ``Zf`` | :c:expr:`float complex` | complex | 8 | \(10) | ++--------+--------------------------+--------------------+----------------+------------+ +| ``Zd`` | :c:expr:`double complex` | complex | 16 | \(10) | ++--------+--------------------------+--------------------+----------------+------------+ | ``s`` | :c:expr:`char[]` | bytes | | \(9) | +--------+--------------------------+--------------------+----------------+------------+ | ``p`` | :c:expr:`char[]` | bytes | | \(8) | +--------+--------------------------+--------------------+----------------+------------+ -| ``P`` | :c:expr:`void \*` | integer | | \(5) | +| ``P`` | :c:expr:`void \*` | int | | \(2), \(5) | +--------+--------------------------+--------------------+----------------+------------+ .. versionchanged:: 3.3 @@ -280,6 +285,15 @@ platform-dependent. .. versionchanged:: 3.14 Added support for the ``'F'`` and ``'D'`` formats. +.. versionchanged:: 3.15 + Added support for the ``'Zf'`` and ``'Zd'`` formats. + +.. seealso:: + + The :mod:`array` and :ref:`ctypes ` modules, + as well as third-party modules like `numpy `__, + use similar -- but slightly different -- type codes. + Notes: @@ -311,7 +325,7 @@ Notes: format used by the platform. (5) - The ``'P'`` format character is only available for the native byte ordering + The ``'P'`` type code is only available for the native byte ordering (selected as the default or with the ``'@'`` byte order character). The byte order character ``'='`` chooses to use little- or big-endian ordering based on the host system. The struct module does not interpret this as native @@ -322,7 +336,9 @@ Notes: revision of the `IEEE 754 standard `_. It has a sign bit, a 5-bit exponent and 11-bit precision (with 10 bits explicitly stored), and can represent numbers between approximately ``6.1e-05`` and ``6.5e+04`` - at full precision. This type is not widely supported by C compilers: on a + at full precision. This type is not widely supported by C compilers: + it's available as :c:expr:`_Float16` type, if the compiler supports the Annex H + of the C23 standard. On a typical machine, an unsigned short can be used for storage, but not for math operations. See the Wikipedia page on the `half-precision floating-point format `_ for more information. @@ -331,42 +347,46 @@ Notes: When packing, ``'x'`` inserts one NUL byte. (8) - The ``'p'`` format character encodes a "Pascal string", meaning a short + The ``'p'`` type code encodes a "Pascal string", meaning a short variable-length string stored in a *fixed number of bytes*, given by the count. The first byte stored is the length of the string, or 255, whichever is - smaller. The bytes of the string follow. If the string passed in to + smaller. The bytes of the string follow. If the byte string passed in to :func:`pack` is too long (longer than the count minus 1), only the leading - ``count-1`` bytes of the string are stored. If the string is shorter than + ``count-1`` bytes of the string are stored. If the byte string is shorter than ``count-1``, it is padded with null bytes so that exactly count bytes in all - are used. Note that for :func:`unpack`, the ``'p'`` format character consumes - ``count`` bytes, but that the string returned can never contain more than 255 + are used. Note that for :func:`unpack`, the ``'p'`` type code consumes + ``count`` bytes, but that the :class:`!bytes` object returned can never contain more than 255 bytes. + When packing, arguments of types :class:`bytes` and :class:`bytearray` + are accepted. (9) - For the ``'s'`` format character, the count is interpreted as the length of the - bytes, not a repeat count like for the other format characters; for example, + For the ``'s'`` type code, the count is interpreted as the length of the + byte string, not a repeat count like for the other type codes; for example, ``'10s'`` means a single 10-byte string mapping to or from a single Python byte string, while ``'10c'`` means 10 separate one byte character elements (e.g., ``cccccccccc``) mapping to or from ten different Python byte objects. (See :ref:`struct-examples` for a concrete demonstration of the difference.) - If a count is not given, it defaults to 1. For packing, the string is + If a count is not given, it defaults to 1. For packing, the byte string is truncated or padded with null bytes as appropriate to make it fit. For - unpacking, the resulting bytes object always has exactly the specified number - of bytes. As a special case, ``'0s'`` means a single, empty string (while + unpacking, the resulting :class:`!bytes` object always has exactly the specified number + of bytes. As a special case, ``'0s'`` means a single, empty byte string (while ``'0c'`` means 0 characters). + When packing, arguments of types :class:`bytes` and :class:`bytearray` + are accepted. (10) - For the ``'F'`` and ``'D'`` format characters, the packed representation uses + For the ``'F'`` and ``'D'`` type codes, the packed representation uses the IEEE 754 binary32 and binary64 format for components of the complex number, regardless of the floating-point format used by the platform. - Note that complex types (``F`` and ``D``) are available unconditionally, + Note that complex types (``F``/``Zf`` and ``D``/``Zd``) are available unconditionally, despite complex types being an optional feature in C. As specified in the C11 standard, each complex type is represented by a two-element C array containing, respectively, the real and imaginary parts. -A format character may be preceded by an integral repeat count. For example, +A type code may be preceded by an integral repeat count. For example, the format string ``'4h'`` means exactly the same as ``'hhhh'``. Whitespace characters between formats are ignored; a count and its format must @@ -383,7 +403,7 @@ then :exc:`struct.error` is raised. .. index:: single: ? (question mark); in struct format strings -For the ``'?'`` format character, the return value is either :const:`True` or +For the ``'?'`` type code, the return value is either :const:`True` or :const:`False`. When packing, the truth value of the argument object is used. Either 0 or 1 in the native or standard bool representation will be packed, and any non-zero value will be ``True`` when unpacking. @@ -438,7 +458,7 @@ the result in a named tuple:: >>> Student._make(unpack('<10sHHb', record)) Student(name=b'raymond ', serialnum=4658, school=264, gradelevel=8) -The ordering of format characters may have an impact on size in native +The ordering of type codes may have an impact on size in native mode since padding is implicit. In standard mode, the user is responsible for inserting any desired padding. Note in @@ -479,7 +499,7 @@ at the end, assuming the platform's longs are aligned on 4-byte boundaries:: Applications ------------ -Two main applications for the :mod:`struct` module exist, data +Two main applications for the :mod:`!struct` module exist, data interchange between Python and C code within an application or another application compiled using the same compiler (:ref:`native formats`), and data interchange between applications using agreed upon data layout @@ -496,7 +516,7 @@ When constructing format strings which mimic native layouts, the compiler and machine architecture determine byte ordering and padding. In such cases, the ``@`` format character should be used to specify native byte ordering and data sizes. Internal pad bytes are normally inserted -automatically. It is possible that a zero-repeat format code will be +automatically. It is possible that a zero-repeat type code will be needed at the end of a format string to round up to the correct byte boundary for proper alignment of consecutive chunks of data. @@ -515,7 +535,7 @@ code solves that problem:: >>> calcsize('@llh0l') 24 -The ``'x'`` format code can be used to specify the repeat, but for +The ``'x'`` type code can be used to specify the repeat, but for native formats it is better to use a zero-repeat format like ``'0l'``. By default, native byte ordering and alignment is used, but it is @@ -571,7 +591,7 @@ below were executed on a 32-bit machine:: Classes ------- -The :mod:`struct` module also defines the following type: +The :mod:`!struct` module also defines the following type: .. class:: Struct(format) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 028a7861f36798a..fe64daa3291d670 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -4,21 +4,18 @@ .. module:: subprocess :synopsis: Subprocess management. -.. moduleauthor:: Peter Åstrand -.. sectionauthor:: Peter Åstrand - **Source code:** :source:`Lib/subprocess.py` -------------- -The :mod:`subprocess` module allows you to spawn new processes, connect to their +The :mod:`!subprocess` module allows you to spawn new processes, connect to their input/output/error pipes, and obtain their return codes. This module intends to replace several older modules and functions:: os.system os.spawn* -Information about how the :mod:`subprocess` module can be used to replace these +Information about how the :mod:`!subprocess` module can be used to replace these modules and functions can be found in the following sections. .. seealso:: @@ -27,8 +24,8 @@ modules and functions can be found in the following sections. .. include:: ../includes/wasm-mobile-notavail.rst -Using the :mod:`subprocess` Module ----------------------------------- +Using the :mod:`!subprocess` Module +----------------------------------- The recommended approach to invoking subprocesses is to use the :func:`run` function for all use cases it can handle. For more advanced use cases, the @@ -630,6 +627,12 @@ functions. the value in ``pw_uid`` will be used. If the value is an integer, it will be passed verbatim. (POSIX only) + .. note:: + + Specifying *user* will not drop existing supplementary group memberships! + The caller must also pass ``extra_groups=()`` to reduce the group membership + of the child process for security purposes. + .. availability:: POSIX .. versionadded:: 3.9 @@ -649,7 +652,7 @@ functions. If specified, *env* must provide any variables required for the program to execute. On Windows, in order to run a `side-by-side assembly`_ the - specified *env* **must** include a valid :envvar:`SystemRoot`. + specified *env* **must** include a valid ``%SystemRoot%``. .. _side-by-side assembly: https://en.wikipedia.org/wiki/Side-by-Side_Assembly @@ -803,14 +806,29 @@ Instances of the :class:`Popen` class have the following methods: .. note:: - When the ``timeout`` parameter is not ``None``, then (on POSIX) the - function is implemented using a busy loop (non-blocking call and short - sleeps). Use the :mod:`asyncio` module for an asynchronous wait: see + When ``timeout`` is not ``None`` and the platform supports it, an + efficient event-driven mechanism is used to wait for process termination: + + - Linux >= 5.3 uses :func:`os.pidfd_open` + :func:`select.poll` + - macOS and other BSD variants use :func:`select.kqueue` + + ``KQ_FILTER_PROC`` + ``KQ_NOTE_EXIT`` + - Windows uses ``WaitForSingleObject`` + + If none of these mechanisms are available, the function falls back to a + busy loop (non-blocking call and short sleeps). + + .. note:: + + Use the :mod:`asyncio` module for an asynchronous wait: see :class:`asyncio.create_subprocess_exec`. .. versionchanged:: 3.3 *timeout* was added. + .. versionchanged:: 3.15 + if *timeout* is not ``None``, use efficient event-driven implementation + on Linux >= 5.3 and macOS / BSD. + .. method:: Popen.communicate(input=None, timeout=None) Interact with process: Send data to stdin. Read data from stdout and stderr, @@ -831,7 +849,9 @@ Instances of the :class:`Popen` class have the following methods: If the process does not terminate after *timeout* seconds, a :exc:`TimeoutExpired` exception will be raised. Catching this exception and - retrying communication will not lose any output. + retrying communication will not lose any output. Supplying *input* to a + subsequent post-timeout :meth:`communicate` call is in undefined behavior + and may become an error in the future. The child process is not killed if the timeout expires, so in order to cleanup properly a well-behaved application should kill the child process and @@ -844,6 +864,11 @@ Instances of the :class:`Popen` class have the following methods: proc.kill() outs, errs = proc.communicate() + After a call to :meth:`~Popen.communicate` raises :exc:`TimeoutExpired`, do + not call :meth:`~Popen.wait`. Use an additional :meth:`~Popen.communicate` + call to finish handling pipes and populate the :attr:`~Popen.returncode` + attribute. + .. note:: The data read is buffered in memory, so do not use this method if the data @@ -945,6 +970,11 @@ Reassigning them to new values is unsupported: A negative value ``-N`` indicates that the child was terminated by signal ``N`` (POSIX only). + When ``shell=True``, the return code reflects the exit status of the shell + itself (e.g. ``/bin/sh``), which may map signals to codes such as + ``128+N``. See the documentation of the shell (for example, the Bash + manual's Exit Status) for details. + Windows Popen Helpers --------------------- @@ -1034,7 +1064,7 @@ on Windows. Windows Constants ^^^^^^^^^^^^^^^^^ -The :mod:`subprocess` module exposes the following constants. +The :mod:`!subprocess` module exposes the following constants. .. data:: STD_INPUT_HANDLE @@ -1323,8 +1353,8 @@ calls these functions. .. _subprocess-replacements: -Replacing Older Functions with the :mod:`subprocess` Module ------------------------------------------------------------ +Replacing Older Functions with the :mod:`!subprocess` Module +------------------------------------------------------------ In this section, "a becomes b" means that b can be used as a replacement for a. @@ -1340,7 +1370,7 @@ In this section, "a becomes b" means that b can be used as a replacement for a. :attr:`~CalledProcessError.output` attribute of the raised exception. In the following examples, we assume that the relevant functions have already -been imported from the :mod:`subprocess` module. +been imported from the :mod:`!subprocess` module. Replacing :program:`/bin/sh` shell command substitution @@ -1400,7 +1430,7 @@ Notes: * The :func:`os.system` function ignores SIGINT and SIGQUIT signals while the command is running, but the caller must do this separately when - using the :mod:`subprocess` module. + using the :mod:`!subprocess` module. A more realistic example would look like this:: @@ -1473,7 +1503,7 @@ handling consistency are valid for these functions. Return ``(exitcode, output)`` of executing *cmd* in a shell. - Execute the string *cmd* in a shell with :meth:`Popen.check_output` and + Execute the string *cmd* in a shell with :func:`check_output` and return a 2-tuple ``(exitcode, output)``. *encoding* and *errors* are used to decode output; see the notes on :ref:`frequently-used-arguments` for more details. @@ -1584,7 +1614,7 @@ runtime): Disable use of ``posix_spawn()`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -On Linux, :mod:`subprocess` defaults to using the ``vfork()`` system call +On Linux, :mod:`!subprocess` defaults to using the ``vfork()`` system call internally when it is safe to do so rather than ``fork()``. This greatly improves performance. diff --git a/Doc/library/superseded.rst b/Doc/library/superseded.rst index d120c6acf621e3f..b6b2f28d067299b 100644 --- a/Doc/library/superseded.rst +++ b/Doc/library/superseded.rst @@ -1,7 +1,7 @@ .. _superseded: ****************** -Superseded Modules +Superseded modules ****************** The modules described in this chapter have been superseded by other modules @@ -24,3 +24,4 @@ currently no modules in this latter category. :maxdepth: 1 getopt.rst + profile.rst diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 54e19af4bd69a67..95f20b06b5aa1e7 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -8,24 +8,26 @@ -------------- -.. moduleauthor:: Jeremy Hylton -.. sectionauthor:: Benjamin Peterson - - Symbol tables are generated by the compiler from AST just before bytecode is generated. The symbol table is responsible for calculating the scope of every -identifier in the code. :mod:`symtable` provides an interface to examine these +identifier in the code. :mod:`!symtable` provides an interface to examine these tables. Generating Symbol Tables ------------------------ -.. function:: symtable(code, filename, compile_type) +.. function:: symtable(code, filename, compile_type, *, module=None) Return the toplevel :class:`SymbolTable` for the Python source *code*. *filename* is the name of the file containing the code. *compile_type* is like the *mode* argument to :func:`compile`. + The optional argument *module* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax warnings + by module name. + + .. versionadded:: 3.15 + Added the *module* parameter. Examining Symbol Tables @@ -174,61 +176,16 @@ Examining Symbol Tables Return a tuple containing names of :term:`free (closure) variables ` in this function. + .. method:: get_cells() -.. class:: Class - - A namespace of a class. This class inherits from :class:`SymbolTable`. - - .. method:: get_methods() - - Return a tuple containing the names of method-like functions declared - in the class. - - Here, the term 'method' designates *any* function defined in the class - body via :keyword:`def` or :keyword:`async def`. - - Functions defined in a deeper scope (e.g., in an inner class) are not - picked up by :meth:`get_methods`. + Return a tuple containing names of :term:`cell variables ` in this table. - For example: + .. versionadded:: 3.15 - .. testsetup:: symtable.Class.get_methods - import warnings - context = warnings.catch_warnings() - context.__enter__() - warnings.simplefilter("ignore", category=DeprecationWarning) - - .. testcleanup:: symtable.Class.get_methods - - context.__exit__() - - .. doctest:: symtable.Class.get_methods - - >>> import symtable - >>> st = symtable.symtable(''' - ... def outer(): pass - ... - ... class A: - ... def f(): - ... def w(): pass - ... - ... def g(self): pass - ... - ... @classmethod - ... async def h(cls): pass - ... - ... global outer - ... def outer(self): pass - ... ''', 'test', 'exec') - >>> class_A = st.get_children()[2] - >>> class_A.get_methods() - ('f', 'g', 'h') - - Although ``A().f()`` raises :exc:`TypeError` at runtime, ``A.f`` is still - considered as a method-like function. +.. class:: Class - .. deprecated-removed:: 3.14 3.16 + A namespace of a class. This class inherits from :class:`SymbolTable`. .. class:: Symbol @@ -285,6 +242,12 @@ Examining Symbol Tables Return ``True`` if the symbol is referenced in its block, but not assigned to. + .. method:: is_cell() + + Return ``True`` if the symbol is referenced but not assigned in a nested block. + + .. versionadded:: 3.15 + .. method:: is_free_class() Return *True* if a class-scoped symbol is free from @@ -355,7 +318,7 @@ Command-Line Usage .. versionadded:: 3.13 -The :mod:`symtable` module can be executed as a script from the command line. +The :mod:`!symtable` module can be executed as a script from the command line. .. code-block:: sh diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 0f986aa580b3c94..7cca6f2bcdae912 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -10,17 +10,17 @@ .. note:: - :mod:`sys.monitoring` is a namespace within the :mod:`sys` module, - not an independent module, so there is no need to - ``import sys.monitoring``, simply ``import sys`` and then use - ``sys.monitoring``. + :mod:`!sys.monitoring` is a namespace within the :mod:`sys` module, + not an independent module, and ``import sys.monitoring`` would fail + with a :exc:`ModuleNotFoundError`. Instead, simply ``import sys`` + and then use ``sys.monitoring``. This namespace provides access to the functions and constants necessary to activate and control event monitoring. As programs execute, events occur that might be of interest to tools that -monitor execution. The :mod:`sys.monitoring` namespace provides means to +monitor execution. The :mod:`!sys.monitoring` namespace provides means to receive callbacks when events of interest occur. The monitoring API consists of three components: @@ -180,8 +180,8 @@ Local events '''''''''''' Local events are associated with normal execution of the program and happen -at clearly defined locations. All local events can be disabled. -The local events are: +at clearly defined locations. All local events can be disabled +per location. The local events are: * :monitoring-event:`PY_START` * :monitoring-event:`PY_RESUME` @@ -205,6 +205,8 @@ Using :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` events will give much better performance as they can be disabled independently. +.. _monitoring-ancillary-events: + Ancillary events '''''''''''''''' @@ -216,14 +218,17 @@ by another event: The :monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events are controlled by the :monitoring-event:`CALL` event. -:monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events will only be seen if the -corresponding :monitoring-event:`CALL` event is being monitored. +:monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events will only be +seen if the corresponding :monitoring-event:`CALL` event is being monitored. + + +.. _monitoring-event-global: Other events '''''''''''' Other events are not necessarily tied to a specific location in the -program and cannot be individually disabled. +program and cannot be individually disabled per location. The other events that can be monitored are: @@ -231,6 +236,12 @@ The other events that can be monitored are: * :monitoring-event:`PY_UNWIND` * :monitoring-event:`RAISE` * :monitoring-event:`EXCEPTION_HANDLED` +* :monitoring-event:`RERAISE` + +.. versionchanged:: 3.15 + Other events can now be turned on and disabled on a per code object + basis. Returning :data:`DISABLE` from a callback disables the event + for the entire code object (for the current tool). The STOP_ITERATION event @@ -244,8 +255,7 @@ raise an exception unless it would be visible to other code. To allow tools to monitor for real exceptions without slowing down generators and coroutines, the :monitoring-event:`STOP_ITERATION` event is provided. -:monitoring-event:`STOP_ITERATION` can be locally disabled, unlike -:monitoring-event:`RAISE`. +:monitoring-event:`STOP_ITERATION` can be locally disabled. Note that the :monitoring-event:`STOP_ITERATION` event and the :monitoring-event:`RAISE` event for a :exc:`StopIteration` exception are @@ -289,12 +299,13 @@ in Python (see :ref:`c-api-monitoring`). .. function:: get_local_events(tool_id: int, code: CodeType, /) -> int - Returns all the local events for *code* + Returns all the :ref:`local events ` for *code* .. function:: set_local_events(tool_id: int, code: CodeType, event_set: int, /) -> None - Activates all the local events for *code* which are set in *event_set*. - Raises a :exc:`ValueError` if *tool_id* is not in use. + Activates all the :ref:`local events ` for *code* + which are set in *event_set*. Raises a :exc:`ValueError` if *tool_id* is not + in use. Disabling events @@ -305,14 +316,19 @@ Disabling events A special value that can be returned from a callback function to disable events for the current code location. -Local events can be disabled for a specific code location by returning -:data:`sys.monitoring.DISABLE` from a callback function. This does not change -which events are set, or any other code locations for the same event. +:ref:`Local events ` can be disabled for a specific code +location by returning :data:`sys.monitoring.DISABLE` from a callback function. +This does not change which events are set, or any other code locations for the +same event. + +:ref:`Other events ` can be disabled on a per code +object basis by returning :data:`sys.monitoring.DISABLE` from a callback +function. This disables the event for the entire code object (for the current +tool). -Disabling events for specific locations is very important for high -performance monitoring. For example, a program can be run under a -debugger with no overhead if the debugger disables all monitoring -except for a few breakpoints. +Disabling events for specific locations is very important for high performance +monitoring. For example, a program can be run under a debugger with no overhead +if the debugger disables all monitoring except for a few breakpoints. .. function:: restart_events() -> None diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 52f0af31c68726a..d0fe0625deb5133 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -11,6 +11,51 @@ interpreter and to functions that interact strongly with the interpreter. It is always available. Unless explicitly noted otherwise, all variables are read-only. +.. data:: abi_info + + .. versionadded:: 3.15 + + An object containing information about the ABI of the currently running + Python interpreter. + It should include information that affect the CPython ABI in ways that + require a specific build of the interpreter chosen from variants that can + co-exist on a single machine. + For example, it does not encode the base OS (Linux or Windows), but does + include pointer size since some systems support both 32- and 64-bit builds. + The available entries are the same on all platforms; + e.g. *pointer_size* is available even on 64-bit-only architectures. + + The following attributes are available: + + .. attribute:: abi_info.pointer_bits + + The width of pointers in bits, as an integer, + equivalent to ``8 * sizeof(void *)``. + Usually, this is ``32`` or ``64``. + + .. attribute:: abi_info.free_threaded + + A Boolean indicating whether the interpreter was built with + :term:`free threading` support. + This reflects either the presence of the :option:`--disable-gil` + :file:`configure` option (on Unix) + or setting the ``DisableGil`` property (on Windows). + + .. attribute:: abi_info.debug + + A Boolean indicating whether the interpreter was built in + :ref:`debug mode `. + This reflects either the presence of the :option:`--with-pydebug` + :file:`configure` option (on Unix) + or the ``Debug`` configuration (on Windows). + + .. attribute:: abi_info.byteorder + + A string indicating the native byte order, + either ``'big'`` or ``'little'``. + This is the same as the :data:`byteorder` attribute. + + .. data:: abiflags On POSIX systems where Python was built with the standard ``configure`` @@ -515,7 +560,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only in the range 0--127, and produce undefined results otherwise. Some systems have a convention for assigning specific meanings to specific exit codes, but these are generally underdeveloped; Unix programs generally use 2 for command - line syntax errors and 1 for all other kind of errors. If another type of + line syntax errors and 1 for all other kinds of errors. If another type of object is passed, ``None`` is equivalent to passing zero, and any other object is printed to :data:`stderr` and results in an exit code of 1. In particular, ``sys.exit("some error message")`` is a quick way to exit a @@ -523,8 +568,9 @@ always available. Unless explicitly noted otherwise, all variables are read-only Since :func:`exit` ultimately "only" raises an exception, it will only exit the process when called from the main thread, and the exception is not - intercepted. Cleanup actions specified by finally clauses of :keyword:`try` statements - are honored, and it is possible to intercept the exit attempt at an outer level. + intercepted. Cleanup actions specified by :keyword:`finally` clauses of + :keyword:`try` statements are honored, and it is possible to intercept the + exit attempt at an outer level. .. versionchanged:: 3.6 If an error occurs in the cleanup after the Python interpreter @@ -833,7 +879,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. versionchanged:: 3.6 Windows is no longer guaranteed to return ``'mbcs'``. See :pep:`529` - and :func:`_enablelegacywindowsfsencoding` for more information. + for more information. .. versionchanged:: 3.7 Return ``'utf-8'`` if the :ref:`Python UTF-8 Mode ` is @@ -865,6 +911,33 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. versionadded:: 3.11 + +.. function:: get_lazy_imports() + + Returns the current lazy imports mode as a string. + + * ``"normal"``: Only imports explicitly marked with the ``lazy`` keyword + are lazy + * ``"all"``: All top-level imports are potentially lazy + + See also :func:`set_lazy_imports` and :pep:`810`. + + .. versionadded:: 3.15 + + +.. function:: get_lazy_imports_filter() + + Returns the current lazy imports filter function, or ``None`` if no + filter is set. + + The filter function is called for every potentially lazy import to + determine whether it should actually be lazy. See + :func:`set_lazy_imports_filter` for details on the filter function + signature. + + .. versionadded:: 3.15 + + .. function:: getrefcount(object) Return the reference count of the *object*. The count returned is generally one @@ -1130,10 +1203,14 @@ always available. Unless explicitly noted otherwise, all variables are read-only The size of the seed key of the hash algorithm + .. attribute:: hash_info.cutoff + + Cutoff for small string DJBX33A optimization in range ``[1, cutoff)``. + .. versionadded:: 3.2 .. versionchanged:: 3.4 - Added *algorithm*, *hash_bits* and *seed_bits* + Added *algorithm*, *hash_bits*, *seed_bits*, and *cutoff*. .. data:: hexversion @@ -1404,6 +1481,21 @@ always available. Unless explicitly noted otherwise, all variables are read-only They hold the legacy representation of ``sys.last_exc``, as returned from :func:`exc_info` above. + +.. data:: lazy_modules + + A :class:`set` of fully qualified module name strings that have been lazily + imported in the current interpreter but not yet loaded. When a + lazily imported module is accessed for the first time, its name is removed + from this set. + + This attribute is intended for debugging and introspection. + + See also :func:`set_lazy_imports` and :pep:`810`. + + .. versionadded:: 3.15 + + .. data:: maxsize An integer giving the maximum value a variable of type :c:type:`Py_ssize_t` can @@ -1669,6 +1761,61 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. versionadded:: 3.11 + +.. function:: set_lazy_imports(mode) + + Sets the global lazy imports mode. The *mode* parameter must be one of + the following strings: + + * ``"normal"``: Only imports explicitly marked with the ``lazy`` keyword + are lazy + * ``"all"``: All top-level imports become potentially lazy + + This function is intended for advanced users who need to control lazy + imports across their entire application. Library developers should + generally not use this function as it affects the runtime execution of + applications. + + In addition to the mode, lazy imports can be controlled via the filter + provided by :func:`set_lazy_imports_filter`. + + See also :func:`get_lazy_imports` and :pep:`810`. + + .. versionadded:: 3.15 + + +.. function:: set_lazy_imports_filter(filter) + + Sets the lazy imports filter callback. The *filter* parameter must be a + callable or ``None`` to clear the filter. + + The filter function is called for every potentially lazy import to + determine whether it should actually be lazy. It must have the following + signature:: + + def filter(importing_module: str, imported_module: str, + fromlist: tuple[str, ...] | None) -> bool + + Where: + + * *importing_module* is the name of the module doing the import + * *imported_module* is the resolved name of the module being imported + (for example, ``lazy from .spam import eggs`` passes + ``package.spam``) + * *fromlist* is the tuple of names being imported (for ``from ... import`` + statements), or ``None`` for regular imports + + The filter should return ``True`` to allow the import to be lazy, or + ``False`` to force an eager import. + + This is an advanced feature intended for specialized users who need + fine-grained control over lazy import behavior. + + See also :func:`get_lazy_imports_filter` and :pep:`810`. + + .. versionadded:: 3.15 + + .. function:: setprofile(profilefunc) .. index:: @@ -1764,7 +1911,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only :func:`settrace` for each thread being debugged or use :func:`threading.settrace`. Trace functions should have three arguments: *frame*, *event*, and - *arg*. *frame* is the current stack frame. *event* is a string: ``'call'``, + *arg*. *frame* is the :ref:`current stack frame `. *event* is a string: ``'call'``, ``'line'``, ``'return'``, ``'exception'`` or ``'opcode'``. *arg* depends on the event type. @@ -1947,6 +2094,9 @@ always available. Unless explicitly noted otherwise, all variables are read-only interpreter is pre-release (alpha, beta, or release candidate) then the local and remote interpreters must be the same exact version. + See :ref:`remote-debugging` for more information about the remote debugging + mechanism. + .. audit-event:: sys.remote_exec pid script_path When the code is executed in the remote process, an @@ -1965,33 +2115,9 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. availability:: Unix, Windows. .. versionadded:: 3.14 + See :pep:`768` for more details. -.. function:: _enablelegacywindowsfsencoding() - - Changes the :term:`filesystem encoding and error handler` to 'mbcs' and - 'replace' respectively, for consistency with versions of Python prior to - 3.6. - - This is equivalent to defining the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` - environment variable before launching Python. - - See also :func:`sys.getfilesystemencoding` and - :func:`sys.getfilesystemencodeerrors`. - - .. availability:: Windows. - - .. note:: - Changing the filesystem encoding after Python startup is risky because - the old fsencoding or paths encoded by the old fsencoding may be cached - somewhere. Use :envvar:`PYTHONLEGACYWINDOWSFSENCODING` instead. - - .. versionadded:: 3.6 - See :pep:`529` for more details. - - .. deprecated-removed:: 3.13 3.16 - Use :envvar:`PYTHONLEGACYWINDOWSFSENCODING` instead. - .. data:: stdin stdout stderr @@ -2152,11 +2278,16 @@ always available. Unless explicitly noted otherwise, all variables are read-only The default hook formats :attr:`!err_msg` and :attr:`!object` as: ``f'{err_msg}: {object!r}'``; use "Exception ignored in" error message - if :attr:`!err_msg` is ``None``. + if :attr:`!err_msg` is ``None``. Similar to the :mod:`traceback` module, + this adds color to exceptions by default. This can be disabled using + :ref:`environment variables `. :func:`sys.unraisablehook` can be overridden to control how unraisable exceptions are handled. + .. versionchanged:: 3.15 + Exceptions are now printed with colorful text. + .. seealso:: :func:`excepthook` which handles uncaught exceptions. @@ -2222,7 +2353,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only The version number used to form registry keys on Windows platforms. This is stored as string resource 1000 in the Python DLL. The value is normally the - major and minor versions of the running Python interpreter. It is provided in the :mod:`sys` + major and minor versions of the running Python interpreter. It is provided in the :mod:`!sys` module for informational purposes; modifying this value has no effect on the registry keys used by Python. diff --git a/Doc/library/sys_path_init.rst b/Doc/library/sys_path_init.rst index a37bb59e7cec760..e6c2cddbe842487 100644 --- a/Doc/library/sys_path_init.rst +++ b/Doc/library/sys_path_init.rst @@ -57,15 +57,19 @@ otherwise they are set to the same value as :data:`sys.base_prefix` and :data:`sys.base_exec_prefix`, respectively. This is used by :ref:`sys-path-init-virtual-environments`. -Finally, the :mod:`site` module is processed and :file:`site-packages` directories -are added to the module search path. A common way to customize the search path is -to create :mod:`sitecustomize` or :mod:`usercustomize` modules as described in -the :mod:`site` module documentation. +Finally, the :mod:`site` module is processed and :file:`site-packages` +directories are added to the module search path. The :envvar:`PYTHONUSERBASE` +environment variable controls where is searched for user site-packages and the +:envvar:`PYTHONNOUSERSITE` environment variable prevents searching for user +site-packages all together. A common way to customize the search path is to +create :mod:`sitecustomize` or :mod:`usercustomize` modules as described in the +:mod:`site` module documentation. .. note:: - Certain command line options may further affect path calculations. - See :option:`-E`, :option:`-I`, :option:`-s` and :option:`-S` for further details. + The command line options :option:`-E`, :option:`-P`, :option:`-I`, + :option:`-S` and :option:`-s` further affect path calculations, see their + documentation for details. .. versionchanged:: 3.14 @@ -96,11 +100,10 @@ Please refer to :mod:`site`'s .. note:: - There are other ways how "virtual environments" could be implemented, this - documentation refers implementations based on the ``pyvenv.cfg`` mechanism, - such as :mod:`venv`. Most virtual environment implementations follow the - model set by :mod:`venv`, but there may be exotic implementations that - diverge from it. + There are other ways "virtual environments" could be implemented. + This documentation refers to implementations based on the ``pyvenv.cfg`` + mechanism, such as :mod:`venv`, that many virtual environment implementations + follow. _pth files ---------- diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 684d14a74c48ab0..8aa912d99ba756e 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -4,9 +4,6 @@ .. module:: sysconfig :synopsis: Python's configuration information -.. moduleauthor:: Tarek Ziadé -.. sectionauthor:: Tarek Ziadé - .. versionadded:: 3.2 **Source code:** :source:`Lib/sysconfig` @@ -16,7 +13,7 @@ -------------- -The :mod:`sysconfig` module provides access to Python's configuration +The :mod:`!sysconfig` module provides access to Python's configuration information like the list of installation paths and the configuration variables relevant for the current platform. @@ -28,7 +25,7 @@ A Python distribution contains a :file:`Makefile` and a :file:`pyconfig.h` header file that are necessary to build both the Python binary itself and third-party C extensions compiled using ``setuptools``. -:mod:`sysconfig` puts all variables found in these files in a dictionary that +:mod:`!sysconfig` puts all variables found in these files in a dictionary that can be accessed using :func:`get_config_vars` or :func:`get_config_var`. Notice that on Windows, it's a much smaller set. @@ -68,7 +65,7 @@ Installation paths ------------------ Python uses an installation scheme that differs depending on the platform and on -the installation options. These schemes are stored in :mod:`sysconfig` under +the installation options. These schemes are stored in :mod:`!sysconfig` under unique identifiers based on the value returned by :const:`os.name`. The schemes are used by package installers to determine where to copy files to. @@ -258,12 +255,12 @@ Path Installation directory Installation path functions --------------------------- -:mod:`sysconfig` provides some functions to determine these installation paths. +:mod:`!sysconfig` provides some functions to determine these installation paths. .. function:: get_scheme_names() Return a tuple containing all schemes currently supported in - :mod:`sysconfig`. + :mod:`!sysconfig`. .. function:: get_default_scheme() @@ -285,7 +282,7 @@ Installation path functions *key* must be either ``"prefix"``, ``"home"``, or ``"user"``. The return value is a scheme name listed in :func:`get_scheme_names`. It - can be passed to :mod:`sysconfig` functions that take a *scheme* argument, + can be passed to :mod:`!sysconfig` functions that take a *scheme* argument, such as :func:`get_paths`. .. versionadded:: 3.10 @@ -313,7 +310,7 @@ Installation path functions .. function:: get_path_names() Return a tuple containing all path names currently supported in - :mod:`sysconfig`. + :mod:`!sysconfig`. .. function:: get_path(name, [scheme, [vars, [expand]]]) @@ -323,7 +320,7 @@ Installation path functions *name* has to be a value from the list returned by :func:`get_path_names`. - :mod:`sysconfig` stores installation paths corresponding to each path name, + :mod:`!sysconfig` stores installation paths corresponding to each path name, for each platform, with variables to be expanded. For instance the *stdlib* path for the *nt* scheme is: ``{base}/Lib``. @@ -382,22 +379,22 @@ Other functions Examples of returned values: - - linux-i586 - - linux-alpha (?) + - linux-x86_64 + - linux-aarch64 - solaris-2.6-sun4u - Windows will return one of: + Windows: - win-amd64 (64-bit Windows on AMD64, aka x86_64, Intel64, and EM64T) - win-arm64 (64-bit Windows on ARM64, aka AArch64) - win32 (all others - specifically, sys.platform is returned) - macOS can return: + POSIX based OS: - - macosx-10.6-ppc - - macosx-10.4-ppc64 - - macosx-10.3-i386 - - macosx-10.4-fat + - linux-x86_64 + - macosx-15.5-arm64 + - macosx-26.0-universal2 (macOS on Apple Silicon or Intel) + - android-24-arm64_v8a For other non-POSIX platforms, currently just returns :data:`sys.platform`. @@ -434,7 +431,7 @@ Other functions Command-line usage ------------------ -You can use :mod:`sysconfig` as a script with Python's *-m* option: +You can use :mod:`!sysconfig` as a script with Python's *-m* option: .. code-block:: shell-session diff --git a/Doc/library/syslog.rst b/Doc/library/syslog.rst index 548898a37bc6ea7..b6bf01240951ebd 100644 --- a/Doc/library/syslog.rst +++ b/Doc/library/syslog.rst @@ -2,7 +2,6 @@ =============================================== .. module:: syslog - :platform: Unix :synopsis: An interface to the Unix syslog library routines. -------------- diff --git a/Doc/library/tabnanny.rst b/Doc/library/tabnanny.rst index 4f61b3dd761400f..570cc7fd93beef9 100644 --- a/Doc/library/tabnanny.rst +++ b/Doc/library/tabnanny.rst @@ -5,11 +5,6 @@ :synopsis: Tool for detecting white space related problems in Python source files in a directory tree. -.. moduleauthor:: Tim Peters -.. sectionauthor:: Peter Funk - -.. rudimentary documentation based on module comments - **Source code:** :source:`Lib/tabnanny.py` -------------- diff --git a/Doc/library/tachyon-flamegraph.png b/Doc/library/tachyon-flamegraph.png new file mode 100644 index 000000000000000..a17cd304f8bb287 Binary files /dev/null and b/Doc/library/tachyon-flamegraph.png differ diff --git a/Doc/library/tachyon-gecko-calltree.png b/Doc/library/tachyon-gecko-calltree.png new file mode 100644 index 000000000000000..71b096940e8fcd0 Binary files /dev/null and b/Doc/library/tachyon-gecko-calltree.png differ diff --git a/Doc/library/tachyon-gecko-flamegraph.png b/Doc/library/tachyon-gecko-flamegraph.png new file mode 100644 index 000000000000000..d427ed85ac04e76 Binary files /dev/null and b/Doc/library/tachyon-gecko-flamegraph.png differ diff --git a/Doc/library/tachyon-gecko-opcodes.png b/Doc/library/tachyon-gecko-opcodes.png new file mode 100644 index 000000000000000..9741eb659120b87 Binary files /dev/null and b/Doc/library/tachyon-gecko-opcodes.png differ diff --git a/Doc/library/tachyon-heatmap-with-opcodes.png b/Doc/library/tachyon-heatmap-with-opcodes.png new file mode 100644 index 000000000000000..5ad67d131548e49 Binary files /dev/null and b/Doc/library/tachyon-heatmap-with-opcodes.png differ diff --git a/Doc/library/tachyon-heatmap.png b/Doc/library/tachyon-heatmap.png new file mode 100644 index 000000000000000..47ac1119f4e5725 Binary files /dev/null and b/Doc/library/tachyon-heatmap.png differ diff --git a/Doc/library/tachyon-live-mode-1.gif b/Doc/library/tachyon-live-mode-1.gif new file mode 100644 index 000000000000000..2d58e807d6a7c1d Binary files /dev/null and b/Doc/library/tachyon-live-mode-1.gif differ diff --git a/Doc/library/tachyon-live-mode-2.gif b/Doc/library/tachyon-live-mode-2.gif new file mode 100644 index 000000000000000..bbc2163fe603eb0 Binary files /dev/null and b/Doc/library/tachyon-live-mode-2.gif differ diff --git a/Doc/library/tachyon-pstats.png b/Doc/library/tachyon-pstats.png new file mode 100644 index 000000000000000..d0281ade660914d Binary files /dev/null and b/Doc/library/tachyon-pstats.png differ diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index acaec5b592bf6e9..29a329fdfeab15b 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -4,14 +4,11 @@ .. module:: tarfile :synopsis: Read and write tar-format archive files. -.. moduleauthor:: Lars Gustäbel -.. sectionauthor:: Lars Gustäbel - **Source code:** :source:`Lib/tarfile.py` -------------- -The :mod:`tarfile` module makes it possible to read and write tar +The :mod:`!tarfile` module makes it possible to read and write tar archives, including those using gzip, bz2 and lzma compression. Use the :mod:`zipfile` module to read or write :file:`.zip` files, or the higher-level functions in :ref:`shutil `. @@ -21,6 +18,14 @@ Some facts and figures: * reads and writes :mod:`gzip`, :mod:`bz2`, :mod:`compression.zstd`, and :mod:`lzma` compressed archives if the respective modules are available. + .. + The following paragraph should be similar to ../includes/optional-module.rst + + If any of these :term:`optional modules ` are missing from + your copy of CPython, look for documentation from your distributor (that is, + whoever provided Python to you). + If you are the distributor, see :ref:`optional-module-requirements`. + * read/write support for the POSIX.1-1988 (ustar) format. * read/write support for the GNU tar format including *longname* and *longlink* @@ -137,6 +142,12 @@ Some facts and figures: a Zstandard dictionary used to improve compression of smaller amounts of data. + For modes ``'w:gz'`` and ``'w|gz'``, :func:`tarfile.open` accepts the + keyword argument *mtime* to create a gzip archive header with that mtime. By + default, the mtime is set to the time of creation of the archive. Use + *mtime* ``0`` to generate a compressed stream that does not depend on + creation time, for reproducible output. + For special purposes, there is a second format for *mode*: ``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile` object that processes its data as a stream of blocks. No random seeking will @@ -198,7 +209,7 @@ Some facts and figures: .. versionchanged:: 3.14 The *preset* keyword argument also works for streams. - .. versionchanged:: next + .. versionchanged:: 3.15 The default compression level was reduced to 6 (down from 9). It is the default level used by most compression tools and a better tradeoff between speed and performance. @@ -212,25 +223,25 @@ Some facts and figures: .. function:: is_tarfile(name) - Return :const:`True` if *name* is a tar archive file, that the :mod:`tarfile` + Return :const:`True` if *name* is a tar archive file, that the :mod:`!tarfile` module can read. *name* may be a :class:`str`, file, or file-like object. .. versionchanged:: 3.9 Support for file and file-like objects. -The :mod:`tarfile` module defines the following exceptions: +The :mod:`!tarfile` module defines the following exceptions: .. exception:: TarError - Base class for all :mod:`tarfile` exceptions. + Base class for all :mod:`!tarfile` exceptions. .. exception:: ReadError Is raised when a tar archive is opened, that either cannot be handled by the - :mod:`tarfile` module or is somehow invalid. + :mod:`!tarfile` module or is somehow invalid. .. exception:: CompressionError @@ -294,7 +305,7 @@ The :mod:`tarfile` module defines the following exceptions: The exception that was raised to reject the replacement member is available as :attr:`!BaseException.__context__`. - .. versionadded:: next + .. versionadded:: 3.15 The following constants are available at the module level: @@ -351,7 +362,7 @@ The following constants are available at the module level: Each of the following constants defines a tar archive format that the -:mod:`tarfile` module is able to create. See section :ref:`tar-formats` for +:mod:`!tarfile` module is able to create. See section :ref:`tar-formats` for details. @@ -659,7 +670,7 @@ be finalized; only the internally used file object will be closed. See the it is best practice to only do so in top-level applications or :mod:`site configuration `. To set a global default this way, a filter function needs to be wrapped in - :func:`staticmethod` to prevent injection of a ``self`` argument. + :deco:`staticmethod` to prevent injection of a ``self`` argument. .. versionchanged:: 3.14 @@ -1146,7 +1157,7 @@ reused in custom filters: Note that this filter does not block *all* dangerous archive features. See :ref:`tarfile-further-verification` for details. - .. versionchanged:: next + .. versionchanged:: 3.15 Link targets are now normalized. @@ -1281,7 +1292,7 @@ Command-Line Interface .. versionadded:: 3.4 -The :mod:`tarfile` module provides a simple command-line interface to interact +The :mod:`!tarfile` module provides a simple command-line interface to interact with tar archives. If you want to create a new tar archive, specify its name after the :option:`-c` @@ -1442,7 +1453,7 @@ parameter in :meth:`TarFile.add`:: Supported tar formats --------------------- -There are three tar formats that can be created with the :mod:`tarfile` module: +There are three tar formats that can be created with the :mod:`!tarfile` module: * The POSIX.1-1988 ustar format (:const:`USTAR_FORMAT`). It supports filenames up to a length of at best 256 characters and linknames up to 100 characters. @@ -1451,7 +1462,7 @@ There are three tar formats that can be created with the :mod:`tarfile` module: * The GNU tar format (:const:`GNU_FORMAT`). It supports long filenames and linknames, files bigger than 8 GiB and sparse files. It is the de facto - standard on GNU/Linux systems. :mod:`tarfile` fully supports the GNU tar + standard on GNU/Linux systems. :mod:`!tarfile` fully supports the GNU tar extensions for long names, sparse file support is read-only. * The POSIX.1-2001 pax format (:const:`PAX_FORMAT`). It is the most flexible @@ -1496,7 +1507,7 @@ Unfortunately, there is no way to autodetect the encoding of an archive. The pax format was designed to solve this problem. It stores non-ASCII metadata using the universal character encoding *UTF-8*. -The details of character conversion in :mod:`tarfile` are controlled by the +The details of character conversion in :mod:`!tarfile` are controlled by the *encoding* and *errors* keyword arguments of the :class:`TarFile` class. *encoding* defines the character encoding to use for the metadata in the diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index f0a81a093b435b6..bf9198e175a0e11 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -4,8 +4,6 @@ .. module:: tempfile :synopsis: Generate temporary files and directories. -.. sectionauthor:: Zack Weinberg - **Source code:** :source:`Lib/tempfile.py` .. index:: @@ -225,8 +223,9 @@ The module defines the following user-callable items: properly implements the :const:`os.O_EXCL` flag for :func:`os.open`. The file is readable and writable only by the creating user ID. If the platform uses permission bits to indicate whether a file is executable, - the file is executable by no one. The file descriptor is not inherited - by child processes. + the file is executable by no one. + + The file descriptor is :ref:`not inherited by child processes `. Unlike :func:`TemporaryFile`, the user of :func:`mkstemp` is responsible for deleting the temporary file when done with it. @@ -385,7 +384,7 @@ not surprise other unsuspecting code by changing global API behavior. Examples -------- -Here are some examples of typical usage of the :mod:`tempfile` module:: +Here are some examples of typical usage of the :mod:`!tempfile` module:: >>> import tempfile diff --git a/Doc/library/termios.rst b/Doc/library/termios.rst index 0c6f3059fe71d16..537dfcedd8cf5a3 100644 --- a/Doc/library/termios.rst +++ b/Doc/library/termios.rst @@ -2,7 +2,6 @@ =========================================== .. module:: termios - :platform: Unix :synopsis: POSIX style tty control. .. index:: @@ -38,7 +37,7 @@ The module defines the following functions: items with indices :const:`VMIN` and :const:`VTIME`, which are integers when these fields are defined). The interpretation of the flags and the speeds as well as the indexing in the *cc* array must be done using the symbolic - constants defined in the :mod:`termios` module. + constants defined in the :mod:`!termios` module. .. function:: tcsetattr(fd, when, attributes) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 9fdb21b8badbbf9..4e21e1ded82724c 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -4,10 +4,8 @@ .. module:: test :synopsis: Regression tests package containing the testing suite for Python. -.. sectionauthor:: Brett Cannon - .. note:: - The :mod:`test` package is meant for internal use by Python only. It is + The :mod:`!test` package is meant for internal use by Python only. It is documented for the benefit of the core developers of Python. Any use of this package outside of Python's standard library is discouraged as code mentioned here can change or be removed without notice between releases of @@ -15,12 +13,12 @@ -------------- -The :mod:`test` package contains all regression tests for Python as well as the +The :mod:`!test` package contains all regression tests for Python as well as the modules :mod:`test.support` and :mod:`test.regrtest`. :mod:`test.support` is used to enhance your tests while :mod:`test.regrtest` drives the testing suite. -Each module in the :mod:`test` package whose name starts with ``test_`` is a +Each module in the :mod:`!test` package whose name starts with ``test_`` is a testing suite for a specific module or feature. All new tests should be written using the :mod:`unittest` or :mod:`doctest` module. Some older tests are written using a "traditional" testing style that compares output printed to @@ -38,8 +36,8 @@ written using a "traditional" testing style that compares output printed to .. _writing-tests: -Writing Unit Tests for the :mod:`test` package ----------------------------------------------- +Writing Unit Tests for the :mod:`!test` package +----------------------------------------------- It is preferred that tests that use the :mod:`unittest` module follow a few guidelines. One is to name the test module by starting it with ``test_`` and end @@ -162,12 +160,12 @@ Running tests using the command-line interface .. module:: test.regrtest :synopsis: Drives the regression test suite. -The :mod:`test` package can be run as a script to drive Python's regression +The :mod:`!test` package can be run as a script to drive Python's regression test suite, thanks to the :option:`-m` option: :program:`python -m test`. Under -the hood, it uses :mod:`test.regrtest`; the call :program:`python -m +the hood, it uses :mod:`!test.regrtest`; the call :program:`python -m test.regrtest` used in previous Python versions still works. Running the script by itself automatically starts running all regression tests in the -:mod:`test` package. It does this by finding all modules in the package whose +:mod:`!test` package. It does this by finding all modules in the package whose name starts with ``test_``, importing them, and executing the function :func:`test_main` if present or loading the tests via unittest.TestLoader.loadTestsFromModule if ``test_main`` does not exist. The @@ -175,14 +173,14 @@ names of tests to execute may also be passed to the script. Specifying a single regression test (:program:`python -m test test_spam`) will minimize output and only print whether the test passed or failed. -Running :mod:`test` directly allows what resources are available for +Running :mod:`!test` directly allows what resources are available for tests to use to be set. You do this by using the ``-u`` command-line option. Specifying ``all`` as the value for the ``-u`` option enables all possible resources: :program:`python -m test -uall`. If all but one resource is desired (a more common case), a comma-separated list of resources that are not desired may be listed after ``all``. The command :program:`python -m test -uall,-audio,-largefile` -will run :mod:`test` with all resources except the ``audio`` and +will run :mod:`!test` with all resources except the ``audio`` and ``largefile`` resources. For a list of all resources and more command-line options, run :program:`python -m test -h`. @@ -197,19 +195,19 @@ regression tests. :ref:`controlled using environment variables `. -:mod:`test.support` --- Utilities for the Python test suite -=========================================================== +:mod:`!test.support` --- Utilities for the Python test suite +============================================================ .. module:: test.support :synopsis: Support for Python's regression test suite. -The :mod:`test.support` module provides support for Python's regression +The :mod:`!test.support` module provides support for Python's regression test suite. .. note:: - :mod:`test.support` is not a public module. It is documented here to help + :mod:`!test.support` is not a public module. It is documented here to help Python developers write tests. The API of this module is subject to change without backwards compatibility concerns between releases. @@ -230,7 +228,7 @@ This module defines the following exceptions: function. -The :mod:`test.support` module defines the following constants: +The :mod:`!test.support` module defines the following constants: .. data:: verbose @@ -363,7 +361,7 @@ The :mod:`test.support` module defines the following constants: .. data:: TEST_SUPPORT_DIR - Set to the top level directory that contains :mod:`test.support`. + Set to the top level directory that contains :mod:`!test.support`. .. data:: TEST_HOME_DIR @@ -438,7 +436,7 @@ The :mod:`test.support` module defines the following constants: Used to test mixed type comparison. -The :mod:`test.support` module defines the following functions: +The :mod:`!test.support` module defines the following functions: .. function:: busy_retry(timeout, err_msg=None, /, *, error=True) @@ -492,6 +490,12 @@ The :mod:`test.support` module defines the following functions: tests. +.. function:: get_resource_value(resource) + + Return the value specified for *resource* (as :samp:`-u {resource}={value}`). + Return ``None`` if *resource* is disabled or no value is specified. + + .. function:: python_is_optimized() Return ``True`` if Python was not built with ``-O0`` or ``-Og``. @@ -714,7 +718,7 @@ The :mod:`test.support` module defines the following functions: .. decorator:: anticipate_failure(condition) A decorator to conditionally mark tests with - :func:`unittest.expectedFailure`. Any use of this decorator should + :deco:`unittest.expectedFailure`. Any use of this decorator should have an associated comment identifying the relevant tracker issue. @@ -851,7 +855,7 @@ The :mod:`test.support` module defines the following functions: Decorator for tests that fill the address space. -.. function:: linked_with_musl() +.. function:: linked_to_musl() Return ``False`` if there is no evidence the interpreter was compiled with ``musl``, otherwise return a version triple, either ``(0, 0, 0)`` if the @@ -1037,7 +1041,7 @@ The :mod:`test.support` module defines the following functions: .. versionadded:: 3.11 -The :mod:`test.support` module defines the following classes: +The :mod:`!test.support` module defines the following classes: .. class:: SuppressCrashReport() @@ -1083,14 +1087,14 @@ The :mod:`test.support` module defines the following classes: Try to match a single stored value (*dv*) with a supplied value (*v*). -:mod:`test.support.socket_helper` --- Utilities for socket tests -================================================================ +:mod:`!test.support.socket_helper` --- Utilities for socket tests +================================================================= .. module:: test.support.socket_helper :synopsis: Support for socket tests. -The :mod:`test.support.socket_helper` module provides support for socket tests. +The :mod:`!test.support.socket_helper` module provides support for socket tests. .. versionadded:: 3.9 @@ -1161,14 +1165,14 @@ The :mod:`test.support.socket_helper` module provides support for socket tests. exceptions. -:mod:`test.support.script_helper` --- Utilities for the Python execution tests -============================================================================== +:mod:`!test.support.script_helper` --- Utilities for the Python execution tests +=============================================================================== .. module:: test.support.script_helper :synopsis: Support for Python's script execution tests. -The :mod:`test.support.script_helper` module provides support for Python's +The :mod:`!test.support.script_helper` module provides support for Python's script execution tests. .. function:: interpreter_requires_environment() @@ -1272,13 +1276,13 @@ script execution tests. path and the archive name for the zip file. -:mod:`test.support.bytecode_helper` --- Support tools for testing correct bytecode generation -============================================================================================= +:mod:`!test.support.bytecode_helper` --- Support tools for testing correct bytecode generation +============================================================================================== .. module:: test.support.bytecode_helper :synopsis: Support tools for testing correct bytecode generation. -The :mod:`test.support.bytecode_helper` module provides support for testing +The :mod:`!test.support.bytecode_helper` module provides support for testing and inspecting bytecode generation. .. versionadded:: 3.9 @@ -1304,13 +1308,13 @@ The module defines the following class: Throws :exc:`AssertionError` if *opname* is found. -:mod:`test.support.threading_helper` --- Utilities for threading tests -====================================================================== +:mod:`!test.support.threading_helper` --- Utilities for threading tests +======================================================================= .. module:: test.support.threading_helper :synopsis: Support for threading tests. -The :mod:`test.support.threading_helper` module provides support for threading tests. +The :mod:`!test.support.threading_helper` module provides support for threading tests. .. versionadded:: 3.10 @@ -1391,13 +1395,13 @@ The :mod:`test.support.threading_helper` module provides support for threading t finished. -:mod:`test.support.os_helper` --- Utilities for os tests -======================================================================== +:mod:`!test.support.os_helper` --- Utilities for os tests +========================================================= .. module:: test.support.os_helper :synopsis: Support for os tests. -The :mod:`test.support.os_helper` module provides support for os tests. +The :mod:`!test.support.os_helper` module provides support for os tests. .. versionadded:: 3.10 @@ -1586,13 +1590,13 @@ The :mod:`test.support.os_helper` module provides support for os tests. wrapped with a wait loop that checks for the existence of the file. -:mod:`test.support.import_helper` --- Utilities for import tests -================================================================ +:mod:`!test.support.import_helper` --- Utilities for import tests +================================================================= .. module:: test.support.import_helper :synopsis: Support for import tests. -The :mod:`test.support.import_helper` module provides support for import tests. +The :mod:`!test.support.import_helper` module provides support for import tests. .. versionadded:: 3.10 @@ -1700,13 +1704,13 @@ The :mod:`test.support.import_helper` module provides support for import tests. will be reverted at the end of the block. -:mod:`test.support.warnings_helper` --- Utilities for warnings tests -==================================================================== +:mod:`!test.support.warnings_helper` --- Utilities for warnings tests +===================================================================== .. module:: test.support.warnings_helper :synopsis: Support for warnings tests. -The :mod:`test.support.warnings_helper` module provides support for warnings tests. +The :mod:`!test.support.warnings_helper` module provides support for warnings tests. .. versionadded:: 3.10 diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index a58b460fef409c6..d12968dee91f3c3 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -4,14 +4,11 @@ .. module:: textwrap :synopsis: Text wrapping and filling -.. moduleauthor:: Greg Ward -.. sectionauthor:: Greg Ward - **Source code:** :source:`Lib/textwrap.py` -------------- -The :mod:`textwrap` module provides some convenience functions, +The :mod:`!textwrap` module provides some convenience functions, as well as :class:`TextWrapper`, the class that does all the work. If you're just wrapping or filling one or two text strings, the convenience functions should be good enough; otherwise, you should use an instance of @@ -102,6 +99,10 @@ functions should be good enough; otherwise, you should use an instance of print(repr(s)) # prints ' hello\n world\n ' print(repr(dedent(s))) # prints 'hello\n world\n' + .. versionchanged:: 3.14 + The :func:`!dedent` function now correctly normalizes blank lines containing + only whitespace characters. Previously, the implementation only normalized + blank lines containing tabs and spaces. .. function:: indent(text, prefix, predicate=None) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index cabb41442f84196..5d9a7b6314b1668 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -191,13 +191,16 @@ This module defines the following functions: Its value may be used to uniquely identify this particular thread system-wide (until the thread terminates, after which the value may be recycled by the OS). - .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD, GNU/kFreeBSD. + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD, GNU/kFreeBSD, Solaris. .. versionadded:: 3.8 .. versionchanged:: 3.13 Added support for GNU/kFreeBSD. + .. versionchanged:: 3.15 + Added support for Solaris. + .. function:: enumerate() @@ -512,7 +515,7 @@ since it is impossible to detect the termination of alien threads. This constructor should always be called with keyword arguments. Arguments are: - *group* should be ``None``; reserved for future extension when a + *group* must be ``None`` as it is reserved for future extension when a :class:`!ThreadGroup` class is implemented. *target* is the callable object to be invoked by the :meth:`run` method. @@ -605,7 +608,7 @@ since it is impossible to detect the termination of alien threads. timeout occurs. When the *timeout* argument is present and not ``None``, it should be a - floating-point number specifying a timeout for the operation in seconds + real number specifying a timeout for the operation in seconds (or fractions thereof). As :meth:`~Thread.join` always returns ``None``, you must call :meth:`~Thread.is_alive` after :meth:`~Thread.join` to decide whether a timeout happened -- if the thread is still alive, the @@ -629,6 +632,9 @@ since it is impossible to detect the termination of alien threads. May raise :exc:`PythonFinalizationError`. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. attribute:: name A string used for identification purposes only. It has no semantics. @@ -761,7 +767,7 @@ All methods are executed atomically. If a call with *blocking* set to ``True`` would block, return ``False`` immediately; otherwise, set the lock to locked and return ``True``. - When invoked with the floating-point *timeout* argument set to a positive + When invoked with the *timeout* argument set to a positive value, block for at most the number of seconds specified by *timeout* and as long as the lock cannot be acquired. A *timeout* argument of ``-1`` specifies an unbounded wait. It is forbidden to specify a *timeout* @@ -780,6 +786,9 @@ All methods are executed atomically. .. versionchanged:: 3.14 Lock acquisition can now be interrupted by signals on Windows. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. method:: release() @@ -860,7 +869,7 @@ call release as many times the lock has been acquired can lead to deadlock. * If no thread owns the lock, acquire the lock and return immediately. * If another thread owns the lock, block until we are able to acquire - lock, or *timeout*, if set to a positive float value. + lock, or *timeout*, if set to a positive value. * If the same thread owns the lock, acquire the lock again, and return immediately. This is the difference between :class:`Lock` and @@ -887,6 +896,9 @@ call release as many times the lock has been acquired can lead to deadlock. .. versionchanged:: 3.2 The *timeout* parameter is new. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. method:: release() @@ -1020,7 +1032,7 @@ item to the buffer only needs to wake up one consumer thread. occurs. Once awakened or timed out, it re-acquires the lock and returns. When the *timeout* argument is present and not ``None``, it should be a - floating-point number specifying a timeout for the operation in seconds + real number specifying a timeout for the operation in seconds (or fractions thereof). When the underlying lock is an :class:`RLock`, it is not released using @@ -1147,6 +1159,9 @@ Semaphores also support the :ref:`context management protocol `. .. versionchanged:: 3.2 The *timeout* parameter is new. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. method:: release(n=1) Release a semaphore, incrementing the internal counter by *n*. When it @@ -1247,7 +1262,7 @@ method. The :meth:`~Event.wait` method blocks until the flag is true. the internal flag did not become true within the given wait time. When the timeout argument is present and not ``None``, it should be a - floating-point number specifying a timeout for the operation in seconds, + real number specifying a timeout for the operation in seconds, or fractions thereof. .. versionchanged:: 3.1 @@ -1421,3 +1436,159 @@ is equivalent to:: Currently, :class:`Lock`, :class:`RLock`, :class:`Condition`, :class:`Semaphore`, and :class:`BoundedSemaphore` objects may be used as :keyword:`with` statement context managers. + + +Iterator synchronization +------------------------ + +By default, Python iterators do not support concurrent access. Most iterators make +no guarantees when accessed simultaneously from multiple threads. Generator +iterators, for example, raise :exc:`ValueError` if one of their iterator methods +is called while the generator is already executing. The tools in this section +allow reliable concurrency support to be added to ordinary iterators and +iterator-producing callables. + +The :class:`serialize_iterator` wrapper lets multiple threads share a single iterator and +take turns consuming from it. While one thread is running ``__next__()``, the +others block until the iterator becomes available. Each value produced by the +underlying iterator is delivered to exactly one caller. + +The :func:`concurrent_tee` function lets multiple threads each receive the full +stream of values from one underlying iterator. It creates independent iterators +that all draw from the same source. Values are buffered until consumed by all +of the derived iterators. + +.. class:: serialize_iterator(iterable) + + Return an iterator wrapper that serializes concurrent calls to + :meth:`~iterator.__next__` using a lock. + + If the wrapped iterator also defines :meth:`~generator.send`, + :meth:`~generator.throw`, or :meth:`~generator.close`, those calls + are serialized as well. + + This makes it possible to share a single iterator, including a generator + iterator, between multiple threads. A lock ensures that calls are handled + one at a time. No values are duplicated or skipped by the wrapper itself. + Each item from the underlying iterator is given to exactly one caller. + + This wrapper does not copy or buffer values. Threads that call + :func:`next` while another thread is already advancing the iterator will + block until the active call completes. + + Example: + + .. code-block:: python + + import threading + + def squares(n): + for x in range(n): + yield x * x + + def consume(name, iterable): + for item in iterable: + print(name, item) + + source = threading.serialize_iterator(squares(5)) + + t1 = threading.Thread(target=consume, args=("left", source)) + t2 = threading.Thread(target=consume, args=("right", source)) + t1.start() + t2.start() + t1.join() + t2.join() + + In this example, each number is printed exactly once, but the work is shared + between the two threads. + + .. versionadded:: 3.15 + + +.. function:: synchronized_iterator(func) + + Wrap an iterator-producing callable so that each iterator it returns is + automatically passed through :class:`serialize_iterator`. + + This is especially useful as a :term:`decorator` for generator functions, + allowing their generator-iterators to be consumed from multiple threads. + + Example: + + .. code-block:: python + + import threading + + @threading.synchronized_iterator + def squares(n): + for x in range(n): + yield x * x + + def consume(name, iterable): + for item in iterable: + print(name, item) + + source = squares(5) + + t1 = threading.Thread(target=consume, args=("left", source)) + t2 = threading.Thread(target=consume, args=("right", source)) + t1.start() + t2.start() + t1.join() + t2.join() + + The returned wrapper preserves the metadata of *func*, such as its name and + wrapped function reference. + + .. versionadded:: 3.15 + + +.. function:: concurrent_tee(iterable, n=2) + + Return *n* independent iterators from a single input *iterable*, with + guaranteed behavior when the derived iterators are consumed concurrently. + + This function is similar to :func:`itertools.tee`, but is intended for cases + where the source iterator may feed consumers running in different threads. + Each returned iterator yields every value from the underlying iterable, in + the same order. + + Internally, values are buffered until every derived iterator has consumed + them. + + The returned iterators share the same underlying synchronization lock. Each + individual derived iterator is intended to be consumed by one thread at a + time. If a single derived iterator must itself be shared by multiple + threads, wrap it with :class:`serialize_iterator`. + + If *n* is ``0``, return an empty tuple. If *n* is negative, raise + :exc:`ValueError`. + + Example: + + .. code-block:: python + + import threading + + def squares(n): + for x in range(n): + yield x * x + + def consume(name, iterable): + for item in iterable: + print(name, item) + + source = squares(5) + left, right = threading.concurrent_tee(source) + + t1 = threading.Thread(target=consume, args=("left", left)) + t2 = threading.Thread(target=consume, args=("right", right)) + t1.start() + t2.start() + t1.join() + t2.join() + + In this example, both consumer threads see the full sequence of squares + from a single generator expression. + + .. versionadded:: 3.15 diff --git a/Doc/library/threadsafety.rst b/Doc/library/threadsafety.rst new file mode 100644 index 000000000000000..a529f7803affbc2 --- /dev/null +++ b/Doc/library/threadsafety.rst @@ -0,0 +1,606 @@ +.. _threadsafety: + +************************ +Thread Safety Guarantees +************************ + +This page documents thread-safety guarantees for built-in types in Python's +free-threaded build. The guarantees described here apply when using Python with +the :term:`GIL` disabled (free-threaded mode). When the GIL is enabled, most +operations are implicitly serialized. + +For general guidance on writing thread-safe code in free-threaded Python, see +:ref:`freethreading-python-howto`. + + +.. _threadsafety-levels: + +Thread safety levels +==================== + +The C API documentation uses the following levels to describe the thread +safety guarantees of each function. The levels are listed from least to +most safe. + +.. _threadsafety-level-incompatible: + +Incompatible +------------ + +A function or operation that cannot be made safe for concurrent use even +with external synchronization. Incompatible code typically accesses +global state in an unsynchronized way and must only be called from a single +thread throughout the program's lifetime. + +Example: a function that modifies process-wide state such as signal handlers +or environment variables, where concurrent calls from any threads, even with +external locking, can conflict with the runtime or other libraries. + +.. _threadsafety-level-compatible: + +Compatible +---------- + +A function or operation that is safe to call from multiple threads +*provided* the caller supplies appropriate external synchronization, for +example by holding a :term:`lock` for the duration of each call. Without +such synchronization, concurrent calls may produce :term:`race conditions +` or :term:`data races `. + +Example: a function that reads from or writes to an object whose internal +state is not protected by a lock. Callers must ensure that no two threads +access the same object at the same time. + +.. _threadsafety-level-distinct: + +Safe on distinct objects +------------------------ + +A function or operation that is safe to call from multiple threads without +external synchronization, as long as each thread operates on a **different** +object. Two threads may call the function at the same time, but they must +not pass the same object (or objects that share underlying state) as +arguments. + +Example: a function that modifies fields of a struct using non-atomic +writes. Two threads can each call the function on their own struct +instance safely, but concurrent calls on the *same* instance require +external synchronization. + +.. _threadsafety-level-shared: + +Safe on shared objects +---------------------- + +A function or operation that is safe for concurrent use on the **same** +object. The implementation uses internal synchronization (such as +:term:`per-object locks ` or +:ref:`critical sections `) to protect shared +mutable state, so callers do not need to supply their own locking. + +Example: :c:func:`PyList_GetItemRef` can be called from multiple threads on the +same :c:type:`PyListObject` - it uses internal synchronization to serialize +access. + +.. _threadsafety-level-atomic: + +Atomic +------ + +A function or operation that appears :term:`atomic ` with +respect to other threads - it executes instantaneously from the perspective +of other threads. This is the strongest form of thread safety. + +Example: :c:func:`PyMutex_IsLocked` performs an atomic read of the mutex +state and can be called from any thread at any time. + + +.. _thread-safety-list: + +Thread safety for list objects +============================== + +Reading a single element from a :class:`list` is +:term:`atomic `: + +.. code-block:: + :class: good + + lst[i] # list.__getitem__ + +The following methods traverse the list and use :term:`atomic ` +reads of each item to perform their function. That means that they may +return results affected by concurrent modifications: + +.. code-block:: + :class: maybe + + item in lst + lst.index(item) + lst.count(item) + +All of the above operations avoid acquiring :term:`per-object locks +`. They do not block concurrent modifications. Other +operations that hold a lock will not block these from observing intermediate +states. + +All other operations from here on block using the :term:`per-object lock`. + +Writing a single item via ``lst[i] = x`` is safe to call from multiple +threads and will not corrupt the list. + +The following operations return new objects and appear +:term:`atomic ` to other threads: + +.. code-block:: + :class: good + + lst1 + lst2 # concatenates two lists into a new list + x * lst # repeats lst x times into a new list + lst.copy() # returns a shallow copy of the list + +The following methods that only operate on a single element with no shifting +required are :term:`atomic `: + +.. code-block:: + :class: good + + lst.append(x) # append to the end of the list, no shifting required + lst.pop() # pop element from the end of the list, no shifting required + +The :meth:`~list.clear` method is also :term:`atomic `. +Other threads cannot observe elements being removed. + +The :meth:`~list.sort` method is not :term:`atomic `. +Other threads cannot observe intermediate states during sorting, but the +list appears empty for the duration of the sort. + +The following operations may allow :term:`lock-free` operations to observe +intermediate states since they modify multiple elements in place: + +.. code-block:: + :class: maybe + + lst.insert(idx, item) # shifts elements + lst.pop(idx) # idx not at the end of the list, shifts elements + lst *= x # copies elements in place + +The :meth:`~list.remove` method may allow concurrent modifications since +element comparison may execute arbitrary Python code (via +:meth:`~object.__eq__`). + +:meth:`~list.extend` is safe to call from multiple threads. However, its +guarantees depend on the iterable passed to it. If it is a :class:`list`, a +:class:`tuple`, a :class:`set`, a :class:`frozenset`, a :class:`dict` or a +:ref:`dictionary view object ` (but not their subclasses), the +``extend`` operation is safe from concurrent modifications to the iterable. +Otherwise, an iterator is created which can be concurrently modified by +another thread. The same applies to inplace concatenation of a list with +other iterables when using ``lst += iterable``. + +Similarly, assigning to a list slice with ``lst[i:j] = iterable`` is safe +to call from multiple threads, but ``iterable`` is only locked when it is +also a :class:`list` (but not its subclasses). + +Operations that involve multiple accesses, as well as iteration, are never +atomic. For example: + +.. code-block:: + :class: bad + + # NOT atomic: read-modify-write + lst[i] = lst[i] + 1 + + # NOT atomic: check-then-act + if lst: + item = lst.pop() + + # NOT thread-safe: iteration while modifying + for item in lst: + process(item) # another thread may modify lst + +Consider external synchronization when sharing :class:`list` instances +across threads. + + +.. _thread-safety-dict: + +Thread safety for dict objects +============================== + +Creating a dictionary with the :class:`dict` constructor is atomic when the +argument to it is a :class:`dict` or a :class:`tuple`. When using the +:meth:`dict.fromkeys` method, dictionary creation is atomic when the +argument is a :class:`dict`, :class:`tuple`, :class:`set` or +:class:`frozenset`. + +The following operations and functions are :term:`lock-free` and +:term:`atomic `. + +.. code-block:: + :class: good + + d[key] # dict.__getitem__ + d.get(key) # dict.get + key in d # dict.__contains__ + len(d) # dict.__len__ + +All other operations from here on hold the :term:`per-object lock`. + +Writing or removing a single item is safe to call from multiple threads +and will not corrupt the dictionary: + +.. code-block:: + :class: good + + d[key] = value # write + del d[key] # delete + d.pop(key) # remove and return + d.popitem() # remove and return last item + d.setdefault(key, v) # insert if missing + +These operations may compare keys using :meth:`~object.__eq__`, which can +execute arbitrary Python code. During such comparisons, the dictionary may +be modified by another thread. For built-in types like :class:`str`, +:class:`int`, and :class:`float`, that implement :meth:`~object.__eq__` in C, +the underlying lock is not released during comparisons and this is not a +concern. + +The following operations return new objects and hold the :term:`per-object lock` +for the duration of the operation: + +.. code-block:: + :class: good + + d.copy() # returns a shallow copy of the dictionary + d | other # merges two dicts into a new dict + d.keys() # returns a new dict_keys view object + d.values() # returns a new dict_values view object + d.items() # returns a new dict_items view object + +The :meth:`~dict.clear` method holds the lock for its duration. Other +threads cannot observe elements being removed. + +The following operations lock both dictionaries. For :meth:`~dict.update` +and ``|=``, this applies only when the other operand is a :class:`dict` +that uses the standard dict iterator (but not subclasses that override +iteration). For equality comparison, this applies to :class:`dict` and +its subclasses: + +.. code-block:: + :class: good + + d.update(other_dict) # both locked when other_dict is a dict + d |= other_dict # both locked when other_dict is a dict + d == other_dict # both locked for dict and subclasses + +All comparison operations also compare values using :meth:`~object.__eq__`, +so for non-built-in types the lock may be released during comparison. + +:meth:`~dict.fromkeys` locks both the new dictionary and the iterable +when the iterable is exactly a :class:`dict`, :class:`set`, or +:class:`frozenset` (not subclasses): + +.. code-block:: + :class: good + + dict.fromkeys(a_dict) # locks both + dict.fromkeys(a_set) # locks both + dict.fromkeys(a_frozenset) # locks both + +When updating from a non-dict iterable, only the target dictionary is +locked. The iterable may be concurrently modified by another thread: + +.. code-block:: + :class: maybe + + d.update(iterable) # iterable is not a dict: only d locked + d |= iterable # iterable is not a dict: only d locked + dict.fromkeys(iterable) # iterable is not a dict/set/frozenset: only result locked + +Operations that involve multiple accesses, as well as iteration, are never +atomic: + +.. code-block:: + :class: bad + + # NOT atomic: read-modify-write + d[key] = d[key] + 1 + + # NOT atomic: check-then-act (TOCTOU) + if key in d: + del d[key] + + # NOT thread-safe: iteration while modifying + for key, value in d.items(): + process(key) # another thread may modify d + +To avoid time-of-check to time-of-use (TOCTOU) issues, use atomic +operations or handle exceptions: + +.. code-block:: + :class: good + + # Use pop() with default instead of check-then-delete + d.pop(key, None) + + # Or handle the exception + try: + del d[key] + except KeyError: + pass + +To safely iterate over a dictionary that may be modified by another +thread, iterate over a copy: + +.. code-block:: + :class: good + + # Make a copy to iterate safely + for key, value in d.copy().items(): + process(key) + +Consider external synchronization when sharing :class:`dict` instances +across threads. + + +.. _thread-safety-set: + +Thread safety for set objects +============================== + +The :func:`len` function is lock-free and :term:`atomic `. + +The following read operation is lock-free. It does not block concurrent +modifications and may observe intermediate states from operations that +hold the per-object lock: + +.. code-block:: + :class: good + + elem in s # set.__contains__ + +This operation may compare elements using :meth:`~object.__eq__`, which can +execute arbitrary Python code. During such comparisons, the set may be +modified by another thread. For built-in types like :class:`str`, +:class:`int`, and :class:`float`, :meth:`!__eq__` does not release the +underlying lock during comparisons and this is not a concern. + +All other operations from here on hold the per-object lock. + +Adding or removing a single element is safe to call from multiple threads +and will not corrupt the set: + +.. code-block:: + :class: good + + s.add(elem) # add element + s.remove(elem) # remove element, raise if missing + s.discard(elem) # remove element if present + s.pop() # remove and return arbitrary element + +These operations also compare elements, so the same :meth:`~object.__eq__` +considerations as above apply. + +The :meth:`~set.copy` method returns a new object and holds the per-object lock +for the duration so that it is always atomic. + +The :meth:`~set.clear` method holds the lock for its duration. Other +threads cannot observe elements being removed. + +The following operations only accept :class:`set` or :class:`frozenset` +as operands and always lock both objects: + +.. code-block:: + :class: good + + s |= other # other must be set/frozenset + s &= other # other must be set/frozenset + s -= other # other must be set/frozenset + s ^= other # other must be set/frozenset + s & other # other must be set/frozenset + s | other # other must be set/frozenset + s - other # other must be set/frozenset + s ^ other # other must be set/frozenset + +:meth:`set.update`, :meth:`set.union`, :meth:`set.intersection` and +:meth:`set.difference` can take multiple iterables as arguments. They all +iterate through all the passed iterables and do the following: + + * :meth:`set.update` and :meth:`set.union` lock both objects only when + the other operand is a :class:`set`, :class:`frozenset`, or :class:`dict`. + * :meth:`set.intersection` and :meth:`set.difference` always try to lock + all objects. + +:meth:`set.symmetric_difference` tries to lock both objects. + +The update variants of the above methods also have some differences between +them: + + * :meth:`set.difference_update` and :meth:`set.intersection_update` try + to lock all objects one-by-one. + * :meth:`set.symmetric_difference_update` only locks the arguments if it is + of type :class:`set`, :class:`frozenset`, or :class:`dict`. + +The following methods always try to lock both objects: + +.. code-block:: + :class: good + + s.isdisjoint(other) # both locked + s.issubset(other) # both locked + s.issuperset(other) # both locked + +Operations that involve multiple accesses, as well as iteration, are never +atomic: + +.. code-block:: + :class: bad + + # NOT atomic: check-then-act + if elem in s: + s.remove(elem) + + # NOT thread-safe: iteration while modifying + for elem in s: + process(elem) # another thread may modify s + +Consider external synchronization when sharing :class:`set` instances +across threads. See :ref:`freethreading-python-howto` for more information. + + +.. _thread-safety-bytearray: + +Thread safety for bytearray objects +=================================== + + The :func:`len` function is lock-free and :term:`atomic `. + + Concatenation and comparisons use the buffer protocol, which prevents + resizing but does not hold the per-object lock. These operations may + observe intermediate states from concurrent modifications: + + .. code-block:: + :class: maybe + + ba + other # may observe concurrent writes + ba == other # may observe concurrent writes + ba < other # may observe concurrent writes + + All other operations from here on hold the per-object lock. + + Reading a single element or slice is safe to call from multiple threads: + + .. code-block:: + :class: good + + ba[i] # bytearray.__getitem__ + ba[i:j] # slice + + The following operations are safe to call from multiple threads and will + not corrupt the bytearray: + + .. code-block:: + :class: good + + ba[i] = x # write single byte + ba[i:j] = values # write slice + ba.append(x) # append single byte + ba.extend(other) # extend with iterable + ba.insert(i, x) # insert single byte + ba.pop() # remove and return last byte + ba.pop(i) # remove and return byte at index + ba.remove(x) # remove first occurrence + ba.reverse() # reverse in place + ba.clear() # remove all bytes + + Slice assignment locks both objects when *values* is a :class:`bytearray`: + + .. code-block:: + :class: good + + ba[i:j] = other_bytearray # both locked + + The following operations return new objects and hold the per-object lock + for the duration: + + .. code-block:: + :class: good + + ba.copy() # returns a shallow copy + ba * n # repeat into new bytearray + + The membership test holds the lock for its duration: + + .. code-block:: + :class: good + + x in ba # bytearray.__contains__ + + All other bytearray methods (such as :meth:`~bytearray.find`, + :meth:`~bytearray.replace`, :meth:`~bytearray.split`, + :meth:`~bytearray.decode`, etc.) hold the per-object lock for their + duration. + + Operations that involve multiple accesses, as well as iteration, are never + atomic: + + .. code-block:: + :class: bad + + # NOT atomic: check-then-act + if x in ba: + ba.remove(x) + + # NOT thread-safe: iteration while modifying + for byte in ba: + process(byte) # another thread may modify ba + + To safely iterate over a bytearray that may be modified by another + thread, iterate over a copy: + + .. code-block:: + :class: good + + # Make a copy to iterate safely + for byte in ba.copy(): + process(byte) + + Consider external synchronization when sharing :class:`bytearray` instances + across threads. See :ref:`freethreading-python-howto` for more information. + + +.. _thread-safety-memoryview: + +Thread safety for memoryview objects +==================================== + +:class:`memoryview` objects provide access to the internal data of an +underlying object without copying. Thread safety depends on both the +memoryview itself and the underlying buffer exporter. + +The memoryview implementation uses atomic operations to track its own +exports in the :term:`free-threaded build`. Creating and +releasing a memoryview are thread-safe. Attribute access (e.g., +:attr:`~memoryview.shape`, :attr:`~memoryview.format`) reads fields that +are immutable for the lifetime of the memoryview, so concurrent reads +are safe as long as the memoryview has not been released. + +However, the actual data accessed through the memoryview is owned by the +underlying object. Concurrent access to this data is only safe if the +underlying object supports it: + +* For immutable objects like :class:`bytes`, concurrent reads through + multiple memoryviews are safe. + +* For mutable objects like :class:`bytearray`, reading and writing the + same memory region from multiple threads without external + synchronization is not safe and may result in data corruption. + Note that even read-only memoryviews of mutable objects do not + prevent data races if the underlying object is modified from + another thread. + +.. code-block:: + :class: bad + + # NOT safe: concurrent writes to the same buffer + data = bytearray(1000) + view = memoryview(data) + # Thread 1: view[0:500] = b'x' * 500 + # Thread 2: view[0:500] = b'y' * 500 + +.. code-block:: + :class: good + + # Safe: use a lock for concurrent access + import threading + lock = threading.Lock() + data = bytearray(1000) + view = memoryview(data) + + with lock: + view[0:500] = b'x' * 500 + +Resizing or reallocating the underlying object (such as calling +:meth:`bytearray.resize`) while a memoryview is exported raises +:exc:`BufferError`. This is enforced regardless of threading. diff --git a/Doc/library/time.rst b/Doc/library/time.rst index df9be68bf4f69d6..a931134331f0a5c 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -189,7 +189,7 @@ Functions .. versionadded:: 3.7 -.. function:: clock_settime(clk_id, time: float) +.. function:: clock_settime(clk_id, time) Set the time of the specified clock *clk_id*. Currently, :data:`CLOCK_REALTIME` is the only accepted value for *clk_id*. @@ -201,6 +201,9 @@ Functions .. versionadded:: 3.3 + .. versionchanged:: 3.15 + Accepts any real number as *time*, not only integer or float. + .. function:: clock_settime_ns(clk_id, time: int) @@ -223,6 +226,9 @@ Functions ``asctime(localtime(secs))``. Locale information is not used by :func:`ctime`. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. + .. function:: get_clock_info(name) @@ -238,8 +244,8 @@ Functions The result has the following attributes: - - *adjustable*: ``True`` if the clock can be changed automatically (e.g. by - a NTP daemon) or manually by the system administrator, ``False`` otherwise + - *adjustable*: ``True`` if the clock can be set to jump forward or backward + in time, ``False`` otherwise. Does not refer to gradual NTP rate adjustments. - *implementation*: The name of the underlying C function used to get the clock value. Refer to :ref:`time-clock-id-constants` for possible values. - *monotonic*: ``True`` if the clock cannot go backward, @@ -258,6 +264,9 @@ Functions :class:`struct_time` object. See :func:`calendar.timegm` for the inverse of this function. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. + .. function:: localtime([secs]) @@ -271,6 +280,9 @@ Functions :c:func:`gmtime` failure. It's common for this to be restricted to years between 1970 and 2038. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. + .. function:: mktime(t) @@ -382,8 +394,7 @@ Functions .. function:: sleep(secs) Suspend execution of the calling thread for the given number of seconds. - The argument may be a floating-point number to indicate a more precise sleep - time. + The argument may be a non-integer to indicate a more precise sleep time. If the sleep is interrupted by a signal and no exception is raised by the signal handler, the sleep is restarted with a recomputed timeout. @@ -396,9 +407,9 @@ Functions On Windows, if *secs* is zero, the thread relinquishes the remainder of its time slice to any other thread that is ready to run. If there are no other threads ready to run, the function returns immediately, and the thread - continues execution. On Windows 8.1 and newer the implementation uses + continues execution. On Windows 10 and newer the implementation uses a `high-resolution timer - `_ + `_ which provides resolution of 100 nanoseconds. If *secs* is zero, ``Sleep(0)`` is used. .. rubric:: Unix implementation @@ -428,6 +439,9 @@ Functions .. versionchanged:: 3.13 Raises an auditing event. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. + .. index:: single: % (percent); datetime format @@ -570,7 +584,7 @@ Functions calculations when the day of the week and the year are specified. Here is an example, a format for dates compatible with that specified in the - :rfc:`2822` Internet email standard. [1]_ :: + :rfc:`5322` Internet email standard. [1]_ :: >>> from time import gmtime, strftime >>> strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()) @@ -935,7 +949,7 @@ These constants are used as parameters for :func:`clock_getres` and .. data:: CLOCK_TAI - `International Atomic Time `_ + `International Atomic Time `_ The system must have a current leap second table in order for this to give the correct answer. PTP or NTP software can maintain a leap second table. @@ -1052,4 +1066,5 @@ Timezone Constants strict reading of the original 1982 :rfc:`822` standard calls for a two-digit year (``%y`` rather than ``%Y``), but practice moved to 4-digit years long before the year 2000. After that, :rfc:`822` became obsolete and the 4-digit year has - been first recommended by :rfc:`1123` and then mandated by :rfc:`2822`. + been first recommended by :rfc:`1123` and then mandated by :rfc:`2822`, + with :rfc:`5322` continuing this requirement. diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 548a3ee05405069..dd574fce09f5fc8 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -19,7 +19,7 @@ See also Tim Peters' introduction to the "Algorithms" chapter in the second edition of *Python Cookbook*, published by O'Reilly. -Basic Examples +Basic examples -------------- The following example shows how the :ref:`timeit-command-line-interface` @@ -56,7 +56,7 @@ repetitions only when the command-line interface is used. In the .. _python-interface: -Python Interface +Python interface ---------------- The module defines three convenience functions and a public class: @@ -143,21 +143,24 @@ The module defines three convenience functions and a public class: timeit.Timer('for i in range(10): oct(i)', 'gc.enable()').timeit() - .. method:: Timer.autorange(callback=None) + .. method:: Timer.autorange(callback=None, target_time=None) Automatically determine how many times to call :meth:`.timeit`. This is a convenience function that calls :meth:`.timeit` repeatedly - so that the total time >= 0.2 second, returning the eventual + so that the total time >= *Timer.target_time* seconds, returning the eventual (number of loops, time taken for that number of loops). It calls :meth:`.timeit` with increasing numbers from the sequence 1, 2, 5, - 10, 20, 50, ... until the time taken is at least 0.2 seconds. + 10, 20, 50, ... until the time taken is at least *target_time* seconds. If *callback* is given and is not ``None``, it will be called after each trial with two arguments: ``callback(number, time_taken)``. .. versionadded:: 3.6 + .. versionchanged:: 3.15 + The optional *target_time* parameter was added. + .. method:: Timer.repeat(repeat=5, number=1000000) @@ -203,7 +206,7 @@ The module defines three convenience functions and a public class: .. _timeit-command-line-interface: -Command-Line Interface +Command-line interface ---------------------- When called as a program from the command line, the following form is used:: @@ -239,6 +242,13 @@ Where the following options are understood: .. versionadded:: 3.5 +.. option:: -t, --target-time=T + + if :option:`--number` is 0, the code will run until it takes at + least this many seconds (default: 0.2) + + .. versionadded:: 3.15 + .. option:: -v, --verbose print raw timing results; repeat for more digits precision @@ -254,7 +264,7 @@ similarly. If :option:`-n` is not given, a suitable number of loops is calculated by trying increasing numbers from the sequence 1, 2, 5, 10, 20, 50, ... until the total -time is at least 0.2 seconds. +time is at least :option:`--target-time` seconds (default: 0.2). :func:`default_timer` measurements can be affected by other programs running on the same machine, so the best thing to do when accurate timing is necessary is @@ -269,6 +279,9 @@ most cases. You can use :func:`time.process_time` to measure CPU time. baseline overhead can be measured by invoking the program without arguments, and it might differ between Python versions. +.. versionadded:: 3.15 + Output is in color by default and can be + :ref:`controlled using environment variables `. .. _timeit-examples: @@ -355,7 +368,7 @@ to test for missing and present object attributes: 0.08588060699912603 -To give the :mod:`timeit` module access to functions you define, you can pass a +To give the :mod:`!timeit` module access to functions you define, you can pass a *setup* parameter which contains an import statement:: def test(): diff --git a/Doc/library/tk.rst b/Doc/library/tk.rst index 0593f8b73ea5456..fa3c7e910ce21f0 100644 --- a/Doc/library/tk.rst +++ b/Doc/library/tk.rst @@ -1,7 +1,7 @@ .. _tkinter: ********************************* -Graphical User Interfaces with Tk +Graphical user interfaces with Tk ********************************* .. index:: @@ -39,6 +39,7 @@ alternative `GUI frameworks and tools ` to be displayed. + .. note:: + + Tk 8.6 added the *command* option. + .. method:: show(**options) Display a message window and wait for the user to select one of the buttons. Then return the symbolic name of the selected button. diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index f284988daf2d4e3..222a4d0128bd8d5 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -4,25 +4,28 @@ .. module:: tkinter :synopsis: Interface to Tcl/Tk for graphical user interfaces -.. moduleauthor:: Guido van Rossum - **Source code:** :source:`Lib/tkinter/__init__.py` -------------- -The :mod:`tkinter` package ("Tk interface") is the standard Python interface to -the Tcl/Tk GUI toolkit. Both Tk and :mod:`tkinter` are available on most Unix +The :mod:`!tkinter` package ("Tk interface") is the standard Python interface to +the Tcl/Tk GUI toolkit. Both Tk and :mod:`!tkinter` are available on most Unix platforms, including macOS, as well as on Windows systems. Running ``python -m tkinter`` from the command line should open a window -demonstrating a simple Tk interface, letting you know that :mod:`tkinter` is +demonstrating a simple Tk interface, letting you know that :mod:`!tkinter` is properly installed on your system, and also showing what version of Tcl/Tk is installed, so you can read the Tcl/Tk documentation specific to that version. -Tkinter supports a range of Tcl/Tk versions, built either with or -without thread support. The official Python binary release bundles Tcl/Tk 8.6 -threaded. See the source code for the :mod:`_tkinter` module -for more information about supported versions. +Tkinter supports a range of Tcl/Tk versions, built either with or without +thread support. +Tcl/Tk 8.5.12 is the minimum supported version; the official Python binary +release bundles Tcl/Tk 9.0. +See the source code for the :mod:`_tkinter` module for more information about +supported versions. + +.. versionchanged:: 3.11 + Support for Tcl/Tk versions older than 8.5.12 was removed. Tkinter is not a thin wrapper, but adds a fair amount of its own logic to make the experience more pythonic. This documentation will concentrate on these @@ -36,6 +39,8 @@ details that are unchanged. Most documentation you will find online still uses the old API and can be woefully outdated. +.. include:: ../includes/optional-module.rst + .. seealso:: * `TkDocs `_ @@ -47,7 +52,7 @@ details that are unchanged. Tcl/Tk Resources: - * `Tk commands `_ + * `Tk commands `_ Comprehensive reference to each of the underlying Tcl/Tk commands used by Tkinter. * `Tcl/Tk Home Page `_ @@ -103,20 +108,20 @@ Ttk bindings are provided in a separate module, :mod:`tkinter.ttk`. Internally, Tk and Ttk use facilities of the underlying operating system, -i.e., Xlib on Unix/X11, Cocoa on macOS, GDI on Windows. +that is, Xlib on Unix/X11, Cocoa on macOS, GDI on Windows. -When your Python application uses a class in Tkinter, e.g., to create a widget, -the :mod:`tkinter` module first assembles a Tcl/Tk command string. It passes that +When your Python application uses a class in Tkinter, for example, to create a widget, +the :mod:`!tkinter` module first assembles a Tcl/Tk command string. It passes that Tcl command string to an internal :mod:`_tkinter` binary module, which then calls the Tcl interpreter to evaluate it. The Tcl interpreter will then call into the Tk and/or Ttk packages, which will in turn make calls to Xlib, Cocoa, or GDI. -Tkinter Modules +Tkinter modules --------------- Support for Tkinter is spread across several modules. Most applications will need the -main :mod:`tkinter` module, as well as the :mod:`tkinter.ttk` module, which provides +main :mod:`!tkinter` module, as well as the :mod:`tkinter.ttk` module, which provides the modern themed widget set and API:: @@ -153,7 +158,7 @@ the modern themed widget set and API:: instead of it being created as an independent toplevel window. *id* must be specified in the same way as the value for the -use option for toplevel widgets (that is, it has a form like that returned by - :meth:`winfo_id`). + :meth:`~Misc.winfo_id`). Note that on some platforms this will only work correctly if *id* refers to a Tk frame or toplevel that has its -container option enabled. @@ -174,13 +179,15 @@ the modern themed widget set and API:: .. attribute:: master - The widget object that contains this widget. For :class:`Tk`, the - *master* is :const:`None` because it is the main window. The terms - *master* and *parent* are similar and sometimes used interchangeably - as argument names; however, calling :meth:`winfo_parent` returns a - string of the widget name whereas :attr:`master` returns the object. - *parent*/*child* reflects the tree-like relationship while - *master*/*slave* reflects the container structure. + The widget object that contains this widget. + For :class:`Tk`, the :attr:`!master` is :const:`None` because it is the + main window. + The terms *master* and *parent* are similar and sometimes used + interchangeably as argument names; however, calling + :meth:`~Misc.winfo_parent` returns a string of the widget name whereas + :attr:`!master` returns the object. + *parent*/*child* reflects the tree-like relationship while *master* (or + *container*)/*content* reflects the container structure. .. attribute:: children @@ -188,21 +195,53 @@ the modern themed widget set and API:: child widget names as the keys and the child instance objects as the values. + .. method:: destroy() + + Destroy this and all descendant widgets and, for the main window, end the + connection to the underlying Tcl interpreter. + + .. method:: loadtk() + + Finish loading and initializing the Tk subsystem. + This is needed only when the interpreter was created without Tk (for + example through :func:`Tcl`); it is called automatically when *useTk* is + true. + + .. method:: readprofile(baseName, className) + + Read and source the user's profile files :file:`.{className}.tcl` and + :file:`.{baseName}.tcl` into the Tcl interpreter, and execute the + corresponding :file:`.{className}.py` and :file:`.{baseName}.py` files. + This is called during initialization; see the description of the + constructor above. + + .. method:: report_callback_exception(exc, val, tb) + + Report a callback exception. + This is called when an exception propagates out of a Tkinter callback; + *exc*, *val* and *tb* are the exception type, value and traceback as + returned by :func:`sys.exc_info`. + The default implementation prints a traceback to :data:`sys.stderr`. + It can be overridden to customize error handling, for example to display + the traceback in a dialog. + .. function:: Tcl(screenName=None, baseName=None, className='Tk', useTk=False) - The :func:`Tcl` function is a factory function which creates an object much like - that created by the :class:`Tk` class, except that it does not initialize the Tk - subsystem. This is most often useful when driving the Tcl interpreter in an - environment where one doesn't want to create extraneous toplevel windows, or - where one cannot (such as Unix/Linux systems without an X server). An object - created by the :func:`Tcl` object can have a Toplevel window created (and the Tk - subsystem initialized) by calling its :meth:`loadtk` method. + The :func:`Tcl` function is a factory function which creates an object much + like that created by the :class:`Tk` class, except that it does not + initialize the Tk subsystem. + This is most often useful when driving the Tcl interpreter in an environment + where one doesn't want to create extraneous toplevel windows, or where one + cannot (such as Unix/Linux systems without an X server). + An object created by the :func:`Tcl` object can have a Toplevel window + created (and the Tk subsystem initialized) by calling its :meth:`~Tk.loadtk` + method. The modules that provide Tk support include: -:mod:`tkinter` +:mod:`!tkinter` Main Tkinter module. :mod:`tkinter.colorchooser` @@ -228,7 +267,7 @@ The modules that provide Tk support include: :mod:`tkinter.ttk` Themed widget set introduced in Tk 8.5, providing modern alternatives - for many of the classic widgets in the main :mod:`tkinter` module. + for many of the classic widgets in the main :mod:`!tkinter` module. Additional modules: @@ -237,29 +276,31 @@ Additional modules: :mod:`_tkinter` A binary module that contains the low-level interface to Tcl/Tk. - It is automatically imported by the main :mod:`tkinter` module, + It is automatically imported by the main :mod:`!tkinter` module, and should never be used directly by application programmers. It is usually a shared library (or DLL), but might in some cases be statically linked with the Python interpreter. :mod:`idlelib` Python's Integrated Development and Learning Environment (IDLE). Based - on :mod:`tkinter`. + on :mod:`!tkinter`. -:mod:`tkinter.constants` +:mod:`!tkinter.constants` Symbolic constants that can be used in place of strings when passing various parameters to Tkinter calls. Automatically imported by the - main :mod:`tkinter` module. + main :mod:`!tkinter` module. :mod:`tkinter.dnd` - (experimental) Drag-and-drop support for :mod:`tkinter`. This will + (experimental) Drag-and-drop support for :mod:`!tkinter`. This will become deprecated when it is replaced with the Tk DND. :mod:`turtle` Turtle graphics in a Tk window. +.. currentmodule:: tkinter + -Tkinter Life Preserver +Tkinter life preserver ---------------------- This section is not designed to be an exhaustive tutorial on either Tk or @@ -274,7 +315,7 @@ find more detailed documentation on them, including in the official Tcl/Tk reference manual. -A Hello World Program +A Hello World program ^^^^^^^^^^^^^^^^^^^^^ We'll start by walking through a "Hello World" application in Tkinter. This @@ -302,19 +343,20 @@ The following line creates a frame widget, which in this case will contain a label and a button we'll create next. The frame is fit inside the root window. -The next line creates a label widget holding a static text string. The -:meth:`grid` method is used to specify the relative layout (position) of the -label within its containing frame widget, similar to how tables in HTML work. +The next line creates a label widget holding a static text string. +The :meth:`~Grid.grid` method is used to specify the relative layout (position) +of the label within its containing frame widget, similar to how tables in HTML +work. -A button widget is then created, and placed to the right of the label. When -pressed, it will call the :meth:`destroy` method of the root window. +A button widget is then created, and placed to the right of the label. +When pressed, it will call the :meth:`~Misc.destroy` method of the root window. Finally, the :meth:`mainloop` method puts everything on the display, and responds to user input until the program terminates. -Important Tk Concepts +Important Tk concepts ^^^^^^^^^^^^^^^^^^^^^ Even this simple program illustrates the following key Tk concepts: @@ -346,7 +388,7 @@ event loop isn't running the event loop, your user interface won't update. -Understanding How Tkinter Wraps Tcl/Tk +Understanding how Tkinter wraps Tcl/Tk ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When your application uses Tkinter's classes and methods, internally Tkinter @@ -392,7 +434,7 @@ by spaces. Without getting into too many details, notice the following: * Operations which are implemented as separate *commands* in Tcl (like ``grid`` or ``destroy``) are represented as *methods* on Tkinter widget objects. As you'll see shortly, at other times Tcl uses what appear to be - method calls on widget objects, which more closely mirror what would is + method calls on widget objects, which more closely mirror what is used in Tkinter. @@ -414,9 +456,9 @@ interactive Python shell or with :func:`print`, can help you identify what you need. To find out what configuration options are available on any widget, call its -:meth:`configure` method, which returns a dictionary containing a variety of -information about each object, including its default and current values. Use -:meth:`keys` to get just the names of each option. +:meth:`~Misc.configure` method, which returns a dictionary containing a variety +of information about each object, including its default and current values. +Use :meth:`~Misc.keys` to get just the names of each option. :: @@ -443,18 +485,20 @@ is helpful. print(set(dir(btn)) - set(dir(frm))) -Navigating the Tcl/Tk Reference Manual +Navigating the Tcl/Tk reference manual ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -As noted, the official `Tk commands `_ -reference manual (man pages) is often the most accurate description of what -specific operations on widgets do. Even when you know the name of the option -or method that you need, you may still have a few places to look. +As noted, the official +`Tk commands `_ reference +manual (man pages) is often the most accurate description of what specific +operations on widgets do. +Even when you know the name of the option or method that you need, you may +still have a few places to look. While all operations in Tkinter are implemented as method calls on widget objects, you've seen that many Tcl/Tk operations appear as commands that take a widget pathname as its first parameter, followed by optional -parameters, e.g. +parameters, for example :: @@ -473,22 +517,23 @@ name of a method to call). In the official Tcl/Tk reference documentation, you'll find most operations -that look like method calls on the man page for a specific widget (e.g., -you'll find the :meth:`invoke` method on the -`ttk::button `_ +that look like method calls on the man page for a specific widget (for example, +you'll find the :meth:`~tkinter.ttk.Button.invoke` method on the +`ttk::button `_ man page), while functions that take a widget as a parameter often have -their own man page (e.g., -`grid `_). +their own man page (for example, +`grid `_). You'll find many common options and methods in the -`options `_ or -`ttk::widget `_ man +`options `_ or +`ttk::widget `_ man pages, while others are found in the man page for a specific widget class. -You'll also find that many Tkinter methods have compound names, e.g., -:func:`winfo_x`, :func:`winfo_height`, :func:`winfo_viewable`. You'd find -documentation for all of these in the -`winfo `_ man page. +You'll also find that many Tkinter methods have compound names, for example, +:meth:`~Misc.winfo_x`, :meth:`~Misc.winfo_height`, +:meth:`~Misc.winfo_viewable`. +You'd find documentation for all of these in the +`winfo `_ man page. .. note:: Somewhat confusingly, there are also methods on all Tkinter widgets @@ -502,7 +547,7 @@ documentation for all of these in the Threading model --------------- -Python and Tcl/Tk have very different threading models, which :mod:`tkinter` +Python and Tcl/Tk have very different threading models, which :mod:`!tkinter` tries to bridge. If you use threads, you may need to be aware of this. A Python interpreter may have many threads associated with it. In Tcl, multiple @@ -510,34 +555,38 @@ threads can be created, but each thread has a separate Tcl interpreter instance associated with it. Threads can also create more than one interpreter instance, though each interpreter instance can be used only by the one thread that created it. -Each :class:`Tk` object created by :mod:`tkinter` contains a Tcl interpreter. +Each :class:`Tk` object created by :mod:`!tkinter` contains a Tcl interpreter. It also keeps track of which thread created that interpreter. Calls to -:mod:`tkinter` can be made from any Python thread. Internally, if a call comes +:mod:`!tkinter` can be made from any Python thread. Internally, if a call comes from a thread other than the one that created the :class:`Tk` object, an event is posted to the interpreter's event queue, and when executed, the result is returned to the calling Python thread. -Tcl/Tk applications are normally event-driven, meaning that after initialization, -the interpreter runs an event loop (i.e. :func:`Tk.mainloop`) and responds to events. -Because it is single-threaded, event handlers must respond quickly, otherwise they -will block other events from being processed. To avoid this, any long-running -computations should not run in an event handler, but are either broken into smaller -pieces using timers, or run in another thread. This is different from many GUI -toolkits where the GUI runs in a completely separate thread from all application -code including event handlers. +Tcl/Tk applications are normally event-driven, meaning that after +initialization, the interpreter runs an event loop (that is, +:meth:`Tk.mainloop `) and responds to events. +Because it is single-threaded, event handlers must respond quickly, otherwise +they will block other events from being processed. +To avoid this, any long-running computations should not run in an event +handler, but are either broken into smaller pieces using timers, or run in +another thread. +This is different from many GUI toolkits where the GUI runs in a completely +separate thread from all application code including event handlers. If the Tcl interpreter is not running the event loop and processing events, any -:mod:`tkinter` calls made from threads other than the one running the Tcl +:mod:`!tkinter` calls made from threads other than the one running the Tcl interpreter will fail. A number of special cases exist: -* Tcl/Tk libraries can be built so they are not thread-aware. In this case, - :mod:`tkinter` calls the library from the originating Python thread, even +* Tcl/Tk libraries built without thread support are now rare: Tcl/Tk 9.0 (the + bundled version) is always thread-aware, so this case only arises with some + older 8.x builds. When the library is not thread-aware, + :mod:`!tkinter` calls the library from the originating Python thread, even if this is different than the thread that created the Tcl interpreter. A global lock ensures only one call occurs at a time. -* While :mod:`tkinter` allows you to create more than one instance of a :class:`Tk` +* While :mod:`!tkinter` allows you to create more than one instance of a :class:`Tk` object (with its own interpreter), all interpreters that are part of the same thread share a common event queue, which gets ugly fast. In practice, don't create more than one instance of :class:`Tk` at a time. Otherwise, it's best to create @@ -548,17 +597,17 @@ A number of special cases exist: or abandon the event loop entirely. If you're doing anything tricky when it comes to events or threads, be aware of these possibilities. -* There are a few select :mod:`tkinter` functions that presently work only when +* There are a few select :mod:`!tkinter` functions that presently work only when called from the thread that created the Tcl interpreter. -Handy Reference +Handy reference --------------- .. _tkinter-setting-options: -Setting Options +Setting options ^^^^^^^^^^^^^^^ Options control things like the color and border width of a widget. Options can @@ -594,11 +643,11 @@ document. Some options don't apply to some kinds of widgets. Whether a given widget responds to a particular option depends on the class of the widget; buttons have a ``command`` option, labels do not. -The options supported by a given widget are listed in that widget's man page, or -can be queried at runtime by calling the :meth:`config` method without -arguments, or by calling the :meth:`keys` method on that widget. The return -value of these calls is a dictionary whose key is the name of the option as a -string (for example, ``'relief'``) and whose values are 5-tuples. +The options supported by a given widget are listed in that widget's man page, +or can be queried at runtime by calling the :meth:`~Misc.config` method without +arguments, or by calling the :meth:`~Misc.keys` method on that widget. +The return value of these calls is a dictionary whose key is the name of the +option as a string (for example, ``'relief'``) and whose values are 5-tuples. Some options, like ``bg`` are synonyms for common options with long names (``bg`` is shorthand for "background"). Passing the ``config()`` method the name @@ -630,30 +679,33 @@ Of course, the dictionary printed will include all the options available and their values. This is meant only as an example. -The Packer +.. _pack-the-packer: + +The packer ^^^^^^^^^^ .. index:: single: packing (widgets) The packer is one of Tk's geometry-management mechanisms. Geometry managers -are used to specify the relative positioning of widgets within their container - -their mutual *master*. In contrast to the more cumbersome *placer* (which is +are used to specify the relative positioning of widgets within their container. +In contrast to the more cumbersome *placer* (which is used less commonly, and we do not cover here), the packer takes qualitative relationship specification - *above*, *to the left of*, *filling*, etc - and works everything out to determine the exact placement coordinates for you. -The size of any *master* widget is determined by the size of the "slave widgets" -inside. The packer is used to control where slave widgets appear inside the -master into which they are packed. You can pack widgets into frames, and frames +The size of any container widget is determined by the size of the "content widgets" +inside. The packer is used to control where content widgets appear inside the +container into which they are packed. You can pack widgets into frames, and frames into other frames, in order to achieve the kind of layout you desire. Additionally, the arrangement is dynamically adjusted to accommodate incremental changes to the configuration, once it is packed. Note that widgets do not appear until they have had their geometry specified -with a geometry manager. It's a common early mistake to leave out the geometry -specification, and then be surprised when the widget is created but nothing -appears. A widget will appear only after it has had, for example, the packer's -:meth:`pack` method applied to it. +with a geometry manager. +It's a common early mistake to leave out the geometry specification, and then +be surprised when the widget is created but nothing appears. +A widget will appear only after it has had, for example, the packer's +:meth:`~Pack.pack` method applied to it. The pack() method can be called with keyword-option/value pairs that control where the widget is to appear within its container, and how it is to behave when @@ -664,32 +716,34 @@ the main application window is resized. Here are some examples:: fred.pack(expand=1) -Packer Options +Packer options ^^^^^^^^^^^^^^ For more extensive information on the packer and the options that it can take, see the man pages and page 183 of John Ousterhout's book. anchor - Anchor type. Denotes where the packer is to place each slave in its parcel. + Anchor type. Denotes where the packer is to place each content in its parcel. expand - Boolean, ``0`` or ``1``. + boolean, ``0`` or ``1``. fill Legal values: ``'x'``, ``'y'``, ``'both'``, ``'none'``. ipadx and ipady - A distance - designating internal padding on each side of the slave widget. + A distance - designating internal padding on each side of the content. padx and pady - A distance - designating external padding on each side of the slave widget. + A distance - designating external padding on each side of the content. side Legal values are: ``'left'``, ``'right'``, ``'top'``, ``'bottom'``. -Coupling Widget Variables +.. _coupling-widget-variables: + +Coupling widget variables ^^^^^^^^^^^^^^^^^^^^^^^^^ The current-value setting of some widgets (like text entry widgets) can be @@ -698,18 +752,19 @@ options are ``variable``, ``textvariable``, ``onvalue``, ``offvalue``, and ``value``. This connection works both ways: if the variable changes for any reason, the widget it's connected to will be updated to reflect the new value. -Unfortunately, in the current implementation of :mod:`tkinter` it is not +Unfortunately, in the current implementation of :mod:`!tkinter` it is not possible to hand over an arbitrary Python variable to a widget through a ``variable`` or ``textvariable`` option. The only kinds of variables for which this works are variables that are subclassed from a class called Variable, -defined in :mod:`tkinter`. +defined in :mod:`!tkinter`. There are many useful subclasses of Variable already defined: :class:`StringVar`, :class:`IntVar`, :class:`DoubleVar`, and -:class:`BooleanVar`. To read the current value of such a variable, call the -:meth:`get` method on it, and to change its value you call the :meth:`!set` -method. If you follow this protocol, the widget will always track the value of -the variable, with no further intervention on your part. +:class:`BooleanVar`. +To read the current value of such a variable, call the :meth:`~Variable.get` +method on it, and to change its value you call the :meth:`!set` method. +If you follow this protocol, the widget will always track the value of the +variable, with no further intervention on your part. For example:: @@ -743,24 +798,29 @@ For example:: myapp = App(root) myapp.mainloop() -The Window Manager +The window manager ^^^^^^^^^^^^^^^^^^ .. index:: single: window manager (widgets) In Tk, there is a utility command, ``wm``, for interacting with the window manager. Options to the ``wm`` command allow you to control things like titles, -placement, icon bitmaps, and the like. In :mod:`tkinter`, these commands have +placement, icon bitmaps, and the like. In :mod:`!tkinter`, these commands have been implemented as methods on the :class:`Wm` class. Toplevel widgets are subclassed from the :class:`Wm` class, and so can call the :class:`Wm` methods directly. To get at the toplevel window that contains a given widget, you can often just -refer to the widget's master. Of course if the widget has been packed inside of -a frame, the master won't represent a toplevel window. To get at the toplevel -window that contains an arbitrary widget, you can call the :meth:`_root` method. -This method begins with an underscore to denote the fact that this function is -part of the implementation, and not an interface to Tk functionality. +refer to the widget's :attr:`~Tk.master`. +Of course if the widget has been packed inside of a frame, the :attr:`!master` +won't represent a toplevel window. +To get at the toplevel window that contains an arbitrary widget, you can call +the :meth:`~Misc.winfo_toplevel` method. +There is also a :meth:`!_root` method; it begins with an underscore to denote +the fact that this function is part of the implementation, and not an interface +to Tk functionality. +It returns the application's root window rather than the nearest enclosing +toplevel. Here are some examples of typical usage:: @@ -784,7 +844,7 @@ Here are some examples of typical usage:: myapp.mainloop() -Tk Option Data Types +Tk option data types ^^^^^^^^^^^^^^^^^^^^ .. index:: single: Tk Option Data Types @@ -794,10 +854,11 @@ anchor ``"s"``, ``"sw"``, ``"w"``, ``"nw"``, and also ``"center"``. bitmap - There are eight built-in, named bitmaps: ``'error'``, ``'gray25'``, - ``'gray50'``, ``'hourglass'``, ``'info'``, ``'questhead'``, ``'question'``, - ``'warning'``. To specify an X bitmap filename, give the full path to the file, - preceded with an ``@``, as in ``"@/usr/contrib/bitmap/gumby.bit"``. + There are ten built-in, named bitmaps: ``'error'``, ``'gray12'``, + ``'gray25'``, ``'gray50'``, ``'gray75'``, ``'hourglass'``, ``'info'``, + ``'questhead'``, ``'question'``, ``'warning'``. To specify an X bitmap + filename, give the full path to the file, preceded with an ``@``, as in + ``"@/usr/contrib/bitmap/gumby.bit"``. boolean You can pass integers 0 or 1 or the strings ``"yes"`` or ``"no"``. @@ -817,7 +878,7 @@ color cursor The standard X cursor names from :file:`cursorfont.h` can be used, without the - ``XC_`` prefix. For example to get a hand cursor (:const:`XC_hand2`), use the + ``XC_`` prefix. For example to get a hand cursor (``XC_hand2``), use the string ``"hand2"``. You can also specify a bitmap and mask file of your own. See page 179 of Ousterhout's book. @@ -829,9 +890,11 @@ distance as ``"3.5i"``. font - Tk uses a list font name format, such as ``{courier 10 bold}``. Font sizes with - positive numbers are measured in points; sizes with negative numbers are - measured in pixels. + Tk uses a font description such as ``{courier 10 bold}``; in + :mod:`!tkinter` this is most naturally passed as a tuple of + ``(family, size, *styles)`` (or as the equivalent string + ``"Courier 10 bold"``). Font sizes with positive numbers are measured in + points; sizes with negative numbers are measured in pixels. geometry This is a string of the form ``widthxheight``, where width and height are @@ -839,8 +902,7 @@ geometry For example: ``fred["geometry"] = "200x100"``. justify - Legal values are the strings: ``"left"``, ``"center"``, ``"right"``, and - ``"fill"``. + Legal values are the strings: ``"left"``, ``"center"``, and ``"right"``. region This is a string with four space-delimited elements, each of which is a legal @@ -849,7 +911,8 @@ region relief Determines what the border style of a widget will be. Legal values are: - ``"raised"``, ``"sunken"``, ``"flat"``, ``"groove"``, and ``"ridge"``. + ``"raised"``, ``"sunken"``, ``"flat"``, ``"groove"``, ``"ridge"``, and + ``"solid"``. scrollcommand This is almost always the :meth:`!set` method of some scrollbar widget, but can @@ -860,7 +923,7 @@ wrap .. _Bindings-and-Events: -Bindings and Events +Bindings and events ^^^^^^^^^^^^^^^^^^^ .. index:: @@ -876,7 +939,10 @@ of the bind method is:: where: sequence - is a string that denotes the target kind of event. (See the + is a string that denotes the target kind of event. Physical events use the + ```` form (for example ``""`` or + ``""``); application-defined virtual events use double angle + brackets, as in ``"<>"``. (See the :manpage:`bind(3tk)` man page, and page 201 of John Ousterhout's book, :title-reference:`Tcl and the Tk Toolkit (2nd edition)`, for details). @@ -922,9 +988,13 @@ they are denoted in Tk, which can be useful when referring to the Tk man pages. +----+---------------------+----+---------------------+ | %y | y | %Y | y_root | +----+---------------------+----+---------------------+ +| %# | serial | %b | num | ++----+---------------------+----+---------------------+ +| %d | detail | %D | delta | ++----+---------------------+----+---------------------+ -The index Parameter +The index parameter ^^^^^^^^^^^^^^^^^^^ A number of widgets require "index" parameters to be passed. These are used to @@ -933,7 +1003,7 @@ Entry widget, or to particular menu items in a Menu widget. Entry widget indexes (index, view index, etc.) Entry widgets have options that refer to character positions in the text being - displayed. You can use these :mod:`tkinter` functions to access these special + displayed. You can use these :mod:`!tkinter` functions to access these special points in text widgets: Text widget indexes @@ -989,7 +1059,7 @@ option (other options are available as well). :meth:`!copy`. The image object can then be used wherever an ``image`` option is supported by -some widget (e.g. labels, buttons, menus). In these cases, Tk will not keep a +some widget (for example, labels, buttons, menus). In these cases, Tk will not keep a reference to the image. When the last Python reference to the image object is deleted, the image data is deleted as well, and Tk will display an empty box wherever the image was used. @@ -1001,7 +1071,7 @@ wherever the image was used. .. _tkinter-file-handlers: -File Handlers +File handlers ------------- Tk allows you to register and unregister a callback function which will be @@ -1047,3 +1117,5307 @@ use raw reads or ``os.read(file.fileno(), maxbytecount)``. EXCEPTION Constants used in the *mask* arguments. + + +Reference +--------- + +.. currentmodule:: tkinter + +This section documents the classes, methods, functions and constants of the +:mod:`!tkinter` module. +Most of them wrap Tcl/Tk commands; consult the official Tcl/Tk manual pages for +the full list of widget options and further details. + +.. exception:: TclError + + The exception raised when a call into the Tcl interpreter fails, for example + when a widget is given an unknown option or an invalid value. + +Base and mixin classes +^^^^^^^^^^^^^^^^^^^^^^ + +.. class:: Misc() + + The :class:`!Misc` class is a mix-in inherited by :class:`Tk` and, through + :class:`BaseWidget`, by every widget. + It provides the large set of methods common to all Tk objects: querying + window information, managing event bindings and the event loop, controlling + the keyboard focus and pointer grabs, accessing the selection, clipboard and + option database, and assorted utility and introspection services. + Because they are inherited, these methods are available on every widget and + on the :class:`Tk` application object, and are documented here once rather + than repeated for each widget. + + .. method:: cget(key) + + Return the current value of the configuration option named *key* for this + widget, as a string. + The expression ``widget[key]`` is equivalent and may be used instead. + + .. method:: config(cnf=None, **kw) + :no-typesetting: + + .. method:: configure(cnf=None, **kw) + + Query or modify the configuration options of the widget. + With no arguments, return a dictionary mapping every available option + name to a tuple describing it (its name, X resource name, X resource + class, default value and current value). + If a single option name is given as a string, return the tuple for just + that option. + If one or more keyword arguments are given, or a dictionary is passed as + *cnf*, set each named option to the corresponding value; the expression + ``widget[key] = value`` sets a single option in the same way. + + :meth:`config` is an alias of :meth:`!configure`. + + .. method:: keys() + + Return a list of the names of all configuration options of this widget. + + .. method:: getboolean(s) + + Interpret the string *s* as a Tcl boolean and return the corresponding + :class:`bool`. + Tcl accepts values such as ``'1'``, ``'0'``, ``'yes'``, ``'no'``, + ``'true'`` and ``'false'``. + Raise :exc:`ValueError` if *s* is not a valid boolean. + + .. method:: getdouble(s) + + Interpret the string *s* as a Tcl floating-point number and return it as + a :class:`float`. + Raise :exc:`ValueError` if *s* is not a valid number. + + .. versionadded:: 3.5 + + + .. method:: getint(s) + + Interpret the string *s* as a Tcl integer and return it as an + :class:`int`. + Raise :exc:`ValueError` if *s* is not a valid integer. + + .. method:: getvar(name='PY_VAR') + + Return the value of the Tcl global variable named *name*. + + .. method:: setvar(name='PY_VAR', value='1') + + Set the Tcl global variable named *name* to *value*. + + The :meth:`!getvar` and :meth:`!setvar` methods give direct access to Tcl + variables. + In most code you will instead use a :class:`Variable` subclass such as + :class:`StringVar` or :class:`IntVar`, which wraps a Tcl variable and + converts its value to and from a Python type. + + .. method:: register(func, subst=None, needcleanup=1) + + Register the Python callable *func* as a Tcl command and return the name + of the new command as a string. + Whenever Tcl invokes that command, *func* is called; if *subst* is given, + it is applied to the command's arguments first. + This is the mechanism used internally to turn Python callbacks into the + command names passed to Tk options such as *command*. + Unless *needcleanup* is false, the command is deleted automatically when + the widget is destroyed. + + .. versionchanged:: 3.13 + The arguments passed to *func* are no longer converted to strings. + + .. method:: deletecommand(name) + + Delete the Tcl command named *name*, such as one previously returned by + :meth:`register`. + + .. method:: nametowidget(name) + + Return the widget instance corresponding to the Tk pathname *name*. + + .. method:: send(interp, cmd, *args) + + Send the Tcl command *cmd*, with the given *args*, to the Tcl interpreter + registered under the name *interp*, and return its result. + This is not available on all platforms. + + .. method:: destroy() + + Destroy this widget and all of its descendant widgets, and delete the Tcl + commands associated with them. + + .. method:: lift(aboveThis=None) + :no-typesetting: + + .. method:: tkraise(aboveThis=None) + + Raise this widget in the stacking order so that it is drawn on top of its + siblings. + If *aboveThis* is given, the widget is moved to be just above it in the + stacking order instead. + + :meth:`lift` is an alias of :meth:`!tkraise`. + + .. method:: lower(belowThis=None) + + Lower this widget in the stacking order so that it is drawn beneath its + siblings. + If *belowThis* is given, the widget is moved to be just below it in the + stacking order instead. + + .. method:: image_names() + + Return the names of all images that currently exist in the Tcl + interpreter. + + .. method:: image_types() + + Return the available image types, such as ``'photo'`` and ``'bitmap'``. + + .. method:: anchor(anchor=None) + :no-typesetting: + + .. method:: grid_anchor(anchor=None) + + Set the anchor that controls where the grid is placed inside this + container when the container is larger than the grid and no row or column + has a non-zero weight. + *anchor* is one of the usual anchor strings, such as ``'nw'`` (the + default) or ``'center'``. + Called with no argument, this method has no effect. + + :meth:`anchor` is an alias of :meth:`!grid_anchor`. + + .. versionadded:: 3.3 + + .. method:: bbox(column=None, row=None, col2=None, row2=None) + :no-typesetting: + + .. method:: grid_bbox(column=None, row=None, col2=None, row2=None) + + Return the bounding box, in pixels, of a region of the grid laid out in + this container, as a 4-tuple ``(xoffset, yoffset, width, height)``. + With no arguments the bounding box of the whole grid is returned. + If *column* and *row* are given, the box spans from the cell at row and + column 0 to that cell; if *col2* and *row2* are also given, it spans from + the cell (*column*, *row*) to the cell (*col2*, *row2*). + + :meth:`bbox` is an alias of :meth:`!grid_bbox`. + + .. method:: columnconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_columnconfigure(index, cnf={}, **kw) + + Query or set the properties of the column (or columns) *index* of the + grid managed by this container. + *index* may be a column number; when setting options it may also be a + list of column numbers, the string ``'all'`` to affect every column, or + a child widget whose occupied columns are affected. + The supported options are: + + *minsize* + The column's minimum size, in pixels. + + *weight* + An integer setting how much of any extra space is apportioned to the + column. + A weight of ``0`` keeps the column at its requested size, and a column + of weight two grows twice as fast as a column of weight one. + + *uniform* + The name of a uniform group. + Columns sharing a non-empty group name are kept in sizes that are + strictly proportional to their weights. + + *pad* + Extra space, in pixels, added to the largest widget in the column when + computing the column's size. + + With a single option name, return that option's value; with no options, + return a dictionary of all of them. + + :meth:`columnconfigure` is an alias of :meth:`!grid_columnconfigure`. + + .. method:: rowconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_rowconfigure(index, cnf={}, **kw) + + Query or set the properties of the row (or rows) *index* of the grid + managed by this container. + *index* is interpreted as for :meth:`grid_columnconfigure`, and the + supported options (*minsize*, *weight*, *uniform* and *pad*) are the + same, applied to a row instead of a column. + + :meth:`rowconfigure` is an alias of :meth:`!grid_rowconfigure`. + + .. method:: grid_location(x, y) + + Return the ``(column, row)`` of the grid cell that contains the pixel at + position (*x*, *y*), given in pixels relative to this container. + For locations above or to the left of the grid, ``-1`` is returned for + the corresponding coordinate. + + .. method:: grid_propagate() + grid_propagate(flag) + + Enable or disable geometry propagation for this container when it manages + its children with the grid geometry manager. + When *flag* is true, the container resizes itself to fit the requested + sizes of its children; when it is false, its size is left under your + control. + Called with no argument, return the current setting as a boolean. + + .. method:: size() + :no-typesetting: + + .. method:: grid_size() + + Return the size of the grid managed by this container as a + ``(columns, rows)`` tuple. + + :meth:`size` is an alias of :meth:`!grid_size`. + + .. method:: grid_slaves(row=None, column=None) + + Return a list of the child widgets managed in this container's grid, most + recently managed first. + If *row* or *column* is given, only the children in that row or column + are returned. + + .. method:: grid_content(row=None, column=None) + + Same as :meth:`grid_slaves`: return the child widgets managed in this + container's grid, optionally restricted to a given *row* or *column*. + + .. versionadded:: 3.15 + + + .. method:: propagate() + propagate(flag) + :no-typesetting: + + .. method:: pack_propagate() + pack_propagate(flag) + + Enable or disable geometry propagation for this container when it manages + its children with the pack geometry manager. + When *flag* is true, the container resizes itself to fit the requested + sizes of its children; when it is false, its size is left under your + control. + Called with no argument, return the current setting as a boolean. + + :meth:`propagate` is an alias of :meth:`!pack_propagate`. + + .. method:: slaves() + :no-typesetting: + + .. method:: pack_slaves() + + Return a list of the child widgets managed by this container with the + pack geometry manager, in packing order. + + :meth:`slaves` is an alias of :meth:`!pack_slaves`. + + .. method:: content() + :no-typesetting: + + .. method:: pack_content() + + Same as :meth:`pack_slaves`: return the child widgets managed by this + container with the pack geometry manager, in packing order. + + :meth:`content` is an alias of :meth:`!pack_content`. + + .. versionadded:: 3.15 + + + .. method:: place_slaves() + + Return a list of the child widgets managed by this container with the + place geometry manager. + + .. method:: place_content() + + Same as :meth:`place_slaves`: return the child widgets managed by this + container with the place geometry manager. + + .. versionadded:: 3.15 + + .. method:: bind(sequence=None, func=None, add=None) + + Bind the event pattern *sequence* on this widget to the callable *func*. + + *sequence* is an event pattern, such as ``''`` (a mouse click) + or ``''``, optionally a concatenation of several such + patterns that must occur shortly after one another. + When the event occurs, *func* is called with an :class:`Event` instance + describing it as its only argument; if *func* returns the string + ``'break'``, no further bindings for the event are invoked. + + If *add* is true, *func* is added to any functions already bound to + *sequence*; otherwise it replaces them. + The binding applies only to this widget. + + :meth:`!bind` returns a string identifier (a *funcid*) that can later be + passed to :meth:`unbind` to remove the binding without leaking the + associated Tcl command. + + If *func* is omitted, return the binding currently associated with + *sequence*; if *sequence* is also omitted, return a list of all the + sequences for which bindings exist on this widget. + + .. method:: bind_class(className, sequence=None, func=None, add=None) + + Like :meth:`bind`, but bind *func* to the binding tag *className* rather + than to a single widget, so that the binding applies to every widget + having that tag. + *className* is usually the name of a widget class, such as ``'Button'``, + in which case the binding affects all widgets of that class. + The set of binding tags for a widget can be inspected and changed with + :meth:`bindtags`. + + The remaining arguments and the return value are as for :meth:`bind`. + + .. method:: bind_all(sequence=None, func=None, add=None) + + Like :meth:`bind`, but bind *func* to the special binding tag ``'all'``, + so that the binding applies to every widget in the application. + + The remaining arguments and the return value are as for :meth:`bind`. + + .. method:: unbind(sequence, funcid=None) + + Remove bindings for the event pattern *sequence* on this widget. + + If *funcid* is given, only the function identified by it (a value + returned from a previous call to :meth:`bind`) is removed, and its + associated Tcl command is deleted. + Otherwise all bindings for *sequence* are destroyed, leaving it unbound. + + .. versionchanged:: 3.13 + If *funcid* is given, only that callback is unbound; other callbacks + bound to *sequence* are kept. + + + .. method:: unbind_class(className, sequence) + + Remove all bindings for the event pattern *sequence* from the binding tag + *className*. + See :meth:`bind_class`. + + .. method:: unbind_all(sequence) + + Remove all bindings for the event pattern *sequence* from the special + binding tag ``'all'``. + See :meth:`bind_all`. + + .. method:: bindtags(tagList=None) + + If *tagList* is omitted, return a tuple of the binding tags associated + with this widget. + When an event occurs in a widget, it is applied to each of the widget's + binding tags in order, and for each tag the most specific matching + binding is executed. + By default a widget has four binding tags: its own pathname, its widget + class, the pathname of its nearest toplevel ancestor, and ``'all'``, in + that order. + + If *tagList* is given, it must be a sequence of strings; the widget's + binding tags are set to its elements, which determines the order in which + bindings are evaluated. + + .. method:: event_add(virtual, *sequences) + + Associate the virtual event *virtual*, whose name has the form + ``'<>'``, with each of the physical event patterns given by + *sequences*, so that the virtual event triggers whenever any of them + occurs. + If *virtual* is already defined, the new sequences are added to its + existing ones. + + .. method:: event_delete(virtual, *sequences) + + Remove each of *sequences* from those associated with the virtual event + *virtual*. + Sequences that are not currently associated with *virtual* are ignored. + If no *sequences* are given, all physical event sequences are removed, so + that *virtual* no longer triggers. + + .. method:: event_generate(sequence, **kw) + + Generate the event *sequence* on this widget and arrange for it to be + processed just as if it had come from the window system. + *sequence* must be a single event pattern, such as ``''`` or + ``'<>'``, not a concatenation of several. + Keyword arguments specify additional fields of the event, for example *x* + and *y* for the pointer position, or *when* to control when the event is + processed; refer to the Tk ``event`` manual page for the full list. + + .. method:: event_info(virtual=None) + + If *virtual* is omitted, return a tuple of all the virtual events that + are currently defined. + If *virtual* is given, return a tuple of the physical event sequences + currently associated with it, or an empty tuple if it is not defined. + + .. method:: after(ms, func=None, *args, **kw) + + Schedule the callable *func* to be called after *ms* milliseconds, with + *args* and *kw* passed to it as positional and keyword arguments. + Return an identifier that can be passed to :meth:`after_cancel` to cancel + the call. + + If *func* is omitted, sleep for *ms* milliseconds instead, processing no + events during that time, and return ``None``. + + .. versionchanged:: 3.10 + *func* can now be any callable object, not only a function. + + .. versionchanged:: 3.14 + Keyword arguments are now passed to *func*. + + + .. method:: after_cancel(id) + + Cancel a callback previously scheduled with :meth:`after` or + :meth:`after_idle`. + *id* must be an identifier returned by one of those methods; passing a + value that is not such an identifier raises :exc:`ValueError`. + If the callback has already run or been cancelled, this has no effect. + + .. versionchanged:: 3.7 + Passing ``None`` (or any false value) as *id* now raises + :exc:`ValueError`. + + + .. method:: after_idle(func, *args, **kw) + + Schedule the callable *func* to be called, with *args* and *kw* passed to + it, when the Tk main loop next becomes idle, that is, when it has no + other events to process. + Return an identifier that can be passed to :meth:`after_cancel` to cancel + the call. + + .. versionchanged:: 3.14 + Keyword arguments are now passed to *func*. + + + .. method:: after_info(id=None) + + If *id* is omitted, return a tuple of the identifiers of all callbacks + currently scheduled with :meth:`after` and :meth:`after_idle` for this + interpreter. + + If *id* is given, it must identify a callback that has not yet run or + been cancelled, and the return value is a tuple ``(script, type)``, where + *script* refers to the function to be called and *type* is either + ``'idle'`` or ``'timer'``. + A :exc:`TclError` is raised if *id* does not exist. + + .. versionadded:: 3.13 + + + .. method:: mainloop(n=0) + + Enter the Tk event loop, which processes events until all windows are + destroyed. + This is normally called once, on the root window, to run the application. + + .. method:: quit() + + Quit the Tcl interpreter, causing :meth:`mainloop` to return. + + .. method:: update() + + Enter the event loop until all pending events, including idle callbacks, + have been processed. + This brings the display up to date and handles any events that are + already queued, then returns. + + .. method:: update_idletasks() + + Enter the event loop until all pending idle callbacks have been called. + This updates the display of windows, for example after geometry changes, + but does not process events caused by the user. + + .. method:: waitvar(name='PY_VAR') + :no-typesetting: + + .. method:: wait_variable(name='PY_VAR') + + Wait until the Tcl variable *name* is modified, continuing to process + events in the meantime so that the application stays responsive. + *name* is usually a :class:`Variable` instance, such as an + :class:`IntVar` or :class:`StringVar`. + + :meth:`waitvar` is an alias of :meth:`!wait_variable`. + + .. method:: wait_window(window=None) + + Wait until *window* is destroyed, continuing to process events in the + meantime. + If *window* is omitted, this widget is used. + This is typically used to wait for the user to finish interacting with a + dialog box. + + .. method:: wait_visibility(window=None) + + Wait until the visibility state of *window* changes, for example when it + first appears on the screen, continuing to process events in the + meantime. + If *window* is omitted, this widget is used. + This is typically used to wait for a newly created window to become + visible before acting on it. + .. method:: focus_set() + :no-typesetting: + + .. method:: focus() + + Direct the keyboard input focus for this widget's display to this widget. + If the application does not currently have the input focus on this + widget's display, the widget is remembered as the focus window for its + top level, and the focus will be redirected to it the next time the + window manager gives the focus to the top level. + :meth:`focus` is an alias of :meth:`!focus_set`. + + .. method:: focus_force() + + Direct the keyboard input focus to this widget even if the application + does not currently have the input focus for the widget's display. + This method should be used sparingly, if at all; normally an application + should wait for the window manager to give it the focus rather than + claiming it. + + .. method:: focus_get() + + Return the widget that currently has the keyboard focus in the + application, or ``None`` if no widget in the application has the focus. + Use :meth:`focus_displayof` to work correctly with several displays. + + .. method:: focus_displayof() + + Return the widget that currently has the keyboard focus on the display + where this widget is located, or ``None`` if no widget in the application + has the focus on that display. + + .. method:: focus_lastfor() + + Return the most recent widget to have had the keyboard focus among all + the widgets in the same top level as this widget; this is the widget that + will receive the focus the next time the window manager gives the focus + to the top level. + If no widget in that top level has ever had the focus, or if the most + recent focus widget has been deleted, the top level itself is returned. + + .. method:: tk_focusFollowsMouse() + + Reconfigure Tk to use an implicit focus model in which the focus is set + to a widget whenever the mouse pointer enters it. + This cannot easily be disabled once enabled. + + .. method:: tk_focusNext() + + Return the next widget after this one in the keyboard traversal order, or + ``None`` if there is none. + The traversal order goes first to the next child, then recursively to the + children of that child, and then to the next sibling higher in the + stacking order. + A widget is skipped if its ``takefocus`` option is set to ``0``. + This method is used in the default bindings for the :kbd:`Tab` key. + + .. method:: tk_focusPrev() + + Return the previous widget before this one in the keyboard traversal + order, or ``None`` if there is none. + See :meth:`tk_focusNext` for how the order is defined. + This method is used in the default bindings for the :kbd:`Shift-Tab` key. + + .. method:: grab_set() + + Set a local grab on this widget. + A grab confines pointer events to this widget and its descendants: while + the pointer is outside the widget's subtree, button presses and releases + and pointer motion are reported to the grab widget, and windows outside + the subtree become insensitive until the grab is released. + A local grab affects only the grabbing application. + Any grab previously set by this application on the widget's display is + automatically released. + Setting a grab is the usual way to make a dialog modal: while the grab is + in effect the user cannot interact with the other windows of the + application. + + .. method:: grab_set_global() + + Set a global grab on this widget. + A global grab is like the local grab set by :meth:`grab_set`, but it + locks out all other applications on the screen, so that only this + widget's subtree is sensitive to pointer events, and it also grabs the + keyboard. + Use with caution: it is easy to render a display unusable with a global + grab, since other applications stop receiving events until it is + released. + + .. method:: grab_release() + + Release the grab on this widget if there is one; otherwise do nothing. + + .. method:: grab_current() + + Return the widget that currently holds the grab in this application for + this widget's display, or ``None`` if there is no such widget. + + .. method:: grab_status() + + Return ``None`` if no grab is currently set on this widget, ``"local"`` + if a local grab is set, or ``"global"`` if a global grab is set. + + .. method:: selection_clear(**kw) + + Clear the X selection, so that no window owns it anymore. + The selection to clear is given by the keyword argument *selection*, an + atom name such as ``'PRIMARY'`` or ``'CLIPBOARD'``; it defaults to + ``PRIMARY``. + The *displayof* keyword argument names a widget that determines the + display on which to operate, and defaults to this widget. + + .. method:: selection_get(**kw) + + Return the contents of the current X selection. + The keyword argument *selection* names the selection and defaults to + ``PRIMARY``. + The keyword argument *type* specifies the form in which the data is to be + returned (the desired conversion target), an atom name such as + ``'STRING'`` or ``'FILE_NAME'``; it defaults to ``STRING``, except on + X11, where ``UTF8_STRING`` is tried first and ``STRING`` is used as a + fallback. + The *displayof* keyword argument names a widget that determines the + display from which to retrieve the selection, and defaults to this + widget. + + .. method:: selection_handle(command, **kw) + + Register *command* as a handler to supply the X selection owned by this + widget when another application requests it. + When the selection is retrieved, *command* is called with two arguments, + the starting character offset and the maximum number of characters to + return, and must return at most that many characters of the selection + starting at that offset; for very long selections it is called repeatedly + with increasing offsets. + The keyword argument *selection* names the selection (default + ``PRIMARY``) and the keyword argument *type* gives the form of the + selection that the handler supplies (such as ``'STRING'`` or + ``'FILE_NAME'``, default ``STRING``). + + .. method:: selection_own(**kw) + + Make this widget the owner of the X selection on its display. + The previous owner, if any, is notified that it has lost the selection. + The keyword argument *selection* names the selection and defaults to + ``PRIMARY``. + + .. method:: selection_own_get(**kw) + + Return the widget in this application that owns the X selection on the + display containing this widget, or ``None`` if no widget in this + application owns the selection. + The keyword argument *selection* names the selection and defaults to + ``PRIMARY``. + The *displayof* keyword argument names a widget that determines the + display to query, and defaults to this widget. + + .. method:: clipboard_append(string, **kw) + + Append *string* to the Tk clipboard and claim ownership of the clipboard + on this widget's display. + Before appending, the clipboard should be emptied with + :meth:`clipboard_clear`; all appends should be completed before returning + to the event loop so that the clipboard is updated atomically. + The keyword argument *type* specifies the form of the data, an atom name + such as ``'STRING'`` or ``'FILE_NAME'`` (default ``STRING``), and the + keyword argument *format* specifies the representation used to transmit + it (default ``STRING``). + The *displayof* keyword argument names a widget that determines the + target display, and defaults to this widget. + The contents can be retrieved with :meth:`clipboard_get` or + :meth:`selection_get`. + + .. method:: clipboard_clear(**kw) + + Claim ownership of the clipboard on this widget's display and remove any + previous contents. + The *displayof* keyword argument names a widget that determines the + target display, and defaults to this widget. + + .. method:: clipboard_get(**kw) + + Retrieve data from the clipboard on this widget's display. + The keyword argument *type* specifies the form in which the data is to be + returned, an atom name such as ``'STRING'`` or ``'FILE_NAME'``; it + defaults to ``STRING``, except on X11, where ``UTF8_STRING`` is tried + first and ``STRING`` is used as a fallback. + The *displayof* keyword argument names a widget that determines the + display, and defaults to the root window of the application. + This is equivalent to ``selection_get(selection= 'CLIPBOARD')``. + + .. method:: option_add(pattern, value, priority=None) + + Add an option to the Tk option database that associates *value* with + *pattern*. + *pattern* consists of names and/or classes separated by asterisks or + dots, in the usual X format. + *priority* is an integer between 0 and 100, or one of the symbolic names + ``'widgetDefault'`` (20), ``'startupFile'`` (40), ``'userDefault'`` (60), + or ``'interactive'`` (80); it defaults to ``interactive``. + + .. method:: option_clear() + + Clear the Tk option database. + Default options from the :envvar:`!RESOURCE_MANAGER` property or the + :file:`.Xdefaults` file are reloaded automatically the next time an + option is added to or removed from the database. + + .. method:: option_get(name, className) + + Return the value of the option matching this widget under *name* and + *className* from the Tk option database, or an empty string if there is + no matching entry. + When several entries match, the one with the highest priority is + returned, and among entries of equal priority the most recently added + one. + + .. method:: option_readfile(fileName, priority=None) + + Read the file named *fileName*, which should have the standard format for + an X resource database such as :file:`.Xdefaults`, and add all the + options it specifies to the Tk option database. + *priority* is interpreted as for :meth:`option_add` and defaults to + ``interactive``. + + .. method:: bell(displayof=0) + + Ring the bell on the display for this widget, using the display's current + bell-related settings, and reset the screen saver for the screen. + If *displayof* is given as a widget, the bell is rung on that widget's + display instead. + + .. method:: tk_setPalette(background, /) + tk_setPalette(*args, **kw) + + Set a new color scheme for all Tk widget elements. + Existing widgets are updated and the option database is changed so that + future widgets use the new colors. + A single color argument is taken as the normal background color, from + which a complete palette is computed. + Alternatively, the arguments may be given as keyword *name*/*value* pairs + naming individual options in the option database. + The recognized option names are ``activeBackground``, + ``activeForeground``, ``background``, ``disabledForeground``, + ``foreground``, ``highlightBackground``, ``highlightColor``, + ``insertBackground``, ``selectColor``, ``selectBackground``, + ``selectForeground``, and ``troughColor``; reasonable defaults are + computed for any that are not specified. + + .. method:: tk_bisque() + + Restore the application's colors to the light brown (bisque) color scheme + used in Tk 3.6 and earlier versions. + Provided for backward compatibility. + + .. method:: tk_strictMotif(boolean=None) + + Query or set whether Tk's look and feel should strictly adhere to Motif. + A true *boolean* value enables strict Motif compliance (for example, no + color change when the mouse passes over a slider). + Return the resulting setting. + .. method:: busy(**kw) + :no-typesetting: + + .. method:: busy_hold(**kw) + :no-typesetting: + + .. method:: tk_busy(**kw) + :no-typesetting: + + .. method:: tk_busy_hold(**kw) + + Make this widget appear busy. + A transparent window is placed in front of the widget, so that it and all + of its descendants in the widget hierarchy are blocked from pointer + events and display a busy cursor. + Normally :meth:`update` should be called immediately afterwards to ensure + that the hold operation is in effect before the application starts its + processing. + + The only supported configuration option is *cursor*, the cursor to be + displayed while the widget is busy; it may have any of the values + accepted by :meth:`!configure`. + + :meth:`busy_hold`, :meth:`busy` and :meth:`tk_busy` are aliases of + :meth:`!tk_busy_hold`. + + .. versionadded:: 3.13 + + + .. method:: busy_configure(cnf=None, **kw) + :no-typesetting: + + .. method:: busy_config(cnf=None, **kw) + :no-typesetting: + + .. method:: tk_busy_config(cnf=None, **kw) + :no-typesetting: + + .. method:: tk_busy_configure(cnf=None, **kw) + + Query or modify the configuration options of the busy window. + The widget must have been previously made busy by :meth:`tk_busy_hold`. + With no arguments, return a dictionary describing all of the available + options; if *cnf* is the name of an option, return a tuple describing + that one option. + Otherwise set the given options to the given values. + Options may have any of the values accepted by :meth:`tk_busy_hold`. + + The option database is referenced through the widget name or class. + For example, if a :class:`Frame` widget named ``frame`` is to be made + busy, the busy cursor can be specified for it by either of the calls:: + + w.option_add('*frame.busyCursor', 'gumby') + w.option_add('*Frame.BusyCursor', 'gumby') + + :meth:`busy_configure`, :meth:`busy_config` and :meth:`tk_busy_config` + are aliases of :meth:`!tk_busy_configure`. + + .. versionadded:: 3.13 + + + .. method:: busy_cget(option) + :no-typesetting: + + .. method:: tk_busy_cget(option) + + Return the current value of the busy configuration *option*. + The widget must have been previously made busy by :meth:`tk_busy_hold`, + and *option* may have any of the values accepted by that method. + + :meth:`busy_cget` is an alias of :meth:`!tk_busy_cget`. + + .. versionadded:: 3.13 + + + .. method:: busy_forget() + :no-typesetting: + + .. method:: tk_busy_forget() + + Make this widget no longer busy, releasing the resources (including the + transparent window) allocated when it was made busy. + User events will again be received by the widget. + These resources are also released when the widget is destroyed. + + :meth:`busy_forget` is an alias of :meth:`!tk_busy_forget`. + + .. versionadded:: 3.13 + + + .. method:: busy_status() + :no-typesetting: + + .. method:: tk_busy_status() + + Return ``True`` if the widget is currently busy, ``False`` otherwise. + + :meth:`busy_status` is an alias of :meth:`!tk_busy_status`. + + .. versionadded:: 3.13 + + + .. method:: busy_current(pattern=None) + :no-typesetting: + + .. method:: tk_busy_current(pattern=None) + + Return a list of widgets that are currently busy. + If *pattern* is given, only busy widgets whose path names match the + pattern are returned. + + :meth:`busy_current` is an alias of :meth:`!tk_busy_current`. + + .. versionadded:: 3.13 + + .. method:: winfo_atom(name, displayof=0) + + Return the integer identifier for the atom whose name is *name*, creating + a new atom if none exists. + If *displayof* is given, the atom is looked up on the display of that + window; otherwise it is looked up on the display of the application's + main window. + + .. method:: winfo_atomname(id, displayof=0) + + Return the textual name for the atom whose integer identifier is *id*. + This is the inverse of :meth:`winfo_atom`. + If *displayof* is given, the identifier is looked up on the display of + that window; otherwise it is looked up on the display of the + application's main window. + + .. method:: winfo_cells() + + Return the number of cells in the colormap for the widget. + + .. method:: winfo_children() + + Return a list containing the widgets that are children of the widget, in + stacking order from lowest to highest. + Toplevel windows are returned as children of their logical parents. + + .. method:: winfo_class() + + Return the class name of the widget. + + .. method:: winfo_colormapfull() + + Return ``True`` if the colormap for the widget is known to be full, + ``False`` otherwise. + + .. method:: winfo_containing(rootX, rootY, displayof=0) + + Return the widget containing the point given by *rootX* and *rootY*, or + ``None`` if no window in this application contains the point. + The coordinates are in screen units in the coordinate system of the root + window. + If *displayof* is given, the coordinates refer to the screen containing + that window; otherwise they refer to the screen of the application's main + window. + + .. method:: winfo_depth() + + Return the depth of the widget, that is, the number of bits per pixel. + + .. method:: winfo_exists() + + Return ``1`` if the widget exists, ``0`` otherwise. + + .. method:: winfo_fpixels(number) + + Return a floating-point value giving the number of pixels in the widget + corresponding to the screen distance *number* (for example, + ``"2.0c"`` or ``"1i"``). + The result may be fractional; for a rounded integer value use + :meth:`winfo_pixels`. + + .. method:: winfo_geometry() + + Return the geometry of the widget, in the form ``widthxheight+x+y``. + All dimensions are in pixels. + + .. method:: winfo_height() + + Return the height of the widget in pixels. + When a window is first created its height is 1 pixel; it is eventually + changed by a geometry manager. + See also :meth:`winfo_reqheight`. + + .. method:: winfo_id() + + Return a low-level platform-specific identifier for the widget. + On Unix this is the X window identifier, and on Windows it is the window + handle. + + .. method:: winfo_interps(displayof=0) + + Return a tuple of the names of all Tcl interpreters currently registered + for a particular display. + If *displayof* is given, the return value refers to the display of that + window; otherwise it refers to the display of the application's main + window. + + .. method:: winfo_ismapped() + + Return ``1`` if the widget is currently mapped, ``0`` otherwise. + + .. method:: winfo_manager() + + Return the name of the geometry manager currently responsible for the + widget, or an empty string if it is not managed by any geometry manager. + + .. method:: winfo_name() + + Return the widget's name within its parent, as opposed to its full path + name. + + .. method:: winfo_parent() + + Return the path name of the widget's parent, or an empty string if the + widget is the main window of the application. + + .. method:: winfo_pathname(id, displayof=0) + + Return the path name of the window whose identifier is *id*. + If *displayof* is given, the identifier is looked up on the display of + that window; otherwise it is looked up on the display of the + application's main window. + + .. method:: winfo_pixels(number) + + Return the number of pixels in the widget corresponding to the screen + distance *number* (for example, ``"2.0c"`` or ``"1i"``). + The result is rounded to the nearest integer; for a fractional result use + :meth:`winfo_fpixels`. + + .. method:: winfo_pointerx() + + Return the pointer's *x* coordinate, in pixels, relative to the screen's + root window (or virtual root, if one is in use). + Return ``-1`` if the pointer is not on the same screen as the widget. + + .. method:: winfo_pointerxy() + + Return the pointer's coordinates as an ``(x, y)`` tuple, in pixels, + relative to the screen's root window (or virtual root, if one is in use). + Both coordinates are ``-1`` if the pointer is not on the same screen as + the widget. + + .. method:: winfo_pointery() + + Return the pointer's *y* coordinate, in pixels, relative to the screen's + root window (or virtual root, if one is in use). + Return ``-1`` if the pointer is not on the same screen as the widget. + + .. method:: winfo_reqheight() + + Return the widget's requested height in pixels. + This is the value used by the widget's geometry manager to compute its + geometry. + + .. method:: winfo_reqwidth() + + Return the widget's requested width in pixels. + This is the value used by the widget's geometry manager to compute its + geometry. + + .. method:: winfo_rgb(color) + + Return an ``(r, g, b)`` tuple of the red, green, and blue intensities, in + the range 0 to 65535, that correspond to *color* in the widget. + *color* may be specified in any of the forms acceptable for a color + option. + + .. method:: winfo_rootx() + + Return the *x* coordinate, in the root window of the screen, of the + upper-left corner of the widget's border (or of the widget itself if it + has no border). + + .. method:: winfo_rooty() + + Return the *y* coordinate, in the root window of the screen, of the + upper-left corner of the widget's border (or of the widget itself if it + has no border). + + .. method:: winfo_screen() + + Return the name of the screen associated with the widget, in the form + ``displayName.screenIndex``. + + .. method:: winfo_screencells() + + Return the number of cells in the default colormap for the widget's + screen. + + .. method:: winfo_screendepth() + + Return the depth of the root window of the widget's screen, that is, the + number of bits per pixel. + + .. method:: winfo_screenheight() + + Return the height of the widget's screen in pixels. + + .. method:: winfo_screenmmheight() + + Return the height of the widget's screen in millimeters. + + .. method:: winfo_screenmmwidth() + + Return the width of the widget's screen in millimeters. + + .. method:: winfo_screenvisual() + + Return the default visual class for the widget's screen, one of + ``"directcolor"``, ``"grayscale"``, ``"pseudocolor"``, ``"staticcolor"``, + ``"staticgray"``, or ``"truecolor"``. + + .. method:: winfo_screenwidth() + + Return the width of the widget's screen in pixels. + + .. method:: winfo_server() + + Return a string containing information about the server for the widget's + display. + The exact format of this string may vary from platform to platform. + + .. method:: winfo_toplevel() + + Return the top-of-hierarchy window containing the widget. + In standard Tk this is always a :class:`Toplevel` widget. + + .. method:: winfo_viewable() + + Return ``1`` if the widget and all of its ancestors up through the + nearest toplevel window are mapped, ``0`` otherwise. + + .. method:: winfo_visual() + + Return the visual class for the widget, one of ``"directcolor"``, + ``"grayscale"``, ``"pseudocolor"``, ``"staticcolor"``, ``"staticgray"``, + or ``"truecolor"``. + + .. method:: winfo_visualid() + + Return the X identifier for the visual for the widget. + + .. method:: winfo_visualsavailable(includeids=False) + + Return a list describing the visuals available for the widget's screen. + Each item consists of a visual class (see :meth:`winfo_visual`) followed + by an integer depth. + If *includeids* is true, the X identifier for the visual is also + included. + + .. method:: winfo_vrootheight() + + Return the height of the virtual root window associated with the widget + if there is one; otherwise return the height of the widget's screen. + + .. method:: winfo_vrootwidth() + + Return the width of the virtual root window associated with the widget if + there is one; otherwise return the width of the widget's screen. + + .. method:: winfo_vrootx() + + Return the *x* offset of the virtual root window associated with the + widget, relative to the root window of its screen. + This is normally zero or negative, and is ``0`` if there is no virtual + root window. + + .. method:: winfo_vrooty() + + Return the *y* offset of the virtual root window associated with the + widget, relative to the root window of its screen. + This is normally zero or negative, and is ``0`` if there is no virtual + root window. + + .. method:: winfo_width() + + Return the width of the widget in pixels. + When a window is first created its width is 1 pixel; it is eventually + changed by a geometry manager. + See also :meth:`winfo_reqwidth`. + + .. method:: winfo_x() + + Return the *x* coordinate, in the widget's parent, of the upper-left + corner of the widget's border (or of the widget itself if it has no + border). + + .. method:: winfo_y() + + Return the *y* coordinate, in the widget's parent, of the upper-left + corner of the widget's border (or of the widget itself if it has no + border). + + .. method:: info_patchlevel() + + Return the Tcl/Tk patch level as a named tuple with the same five fields + as :data:`sys.version_info`: *major*, *minor*, *micro*, *releaselevel* + and *serial*. + *releaselevel* is ``'alpha'``, ``'beta'`` or ``'final'``. + Converting it to a string gives the version in the usual Tcl/Tk notation, + for example ``'9.0.3'`` for a final release or ``'9.1b2'`` for a + pre-release. + + .. versionadded:: 3.11 + + + +.. class:: Wm() + + The :class:`!Wm` mixin provides access to the window manager, allowing an + application to control such things as the title, geometry and icon of a + top-level window, the way it is resized, and how it responds to window + manager protocols. + It is mixed into :class:`Tk` and :class:`Toplevel`, so its methods are + available on every top-level window. + Each method has two equivalent spellings: a short name and a + ``wm_``-prefixed name (for example, :meth:`title` and :meth:`wm_title`). + + .. method:: wm_aspect(minNumer=None, minDenom=None, maxNumer=None, maxDenom=None) + :no-typesetting: + + .. method:: aspect(minNumer=None, minDenom=None, maxNumer=None, maxDenom=None) + + Constrain the aspect ratio (the ratio of width to height) of the window. + If all four arguments are given, the window manager keeps the ratio + between ``minNumer/minDenom`` and ``maxNumer/maxDenom``; passing empty + strings removes any existing restriction. + With no arguments, return a tuple of the four current values, or an empty + string if no aspect restriction is in effect. + :meth:`wm_aspect` is an alias of :meth:`!aspect`. + + .. method:: wm_attributes(*args, return_python_dict=False, **kwargs) + :no-typesetting: + + .. method:: attributes(*args, return_python_dict=False, **kwargs) + + Query or set platform-specific attributes of the window. + With no arguments, return the platform-specific flags and their values; + pass *return_python_dict* as true to get them as a dictionary. + A single option name such as ``'alpha'`` returns the value of that + option, and options are set using keyword arguments (``alpha=0.5``). + + The available attributes differ by platform. + All platforms support: + + *alpha* + The window's opacity, from ``0.0`` (fully transparent) to ``1.0`` + (opaque). + Where transparency is unsupported the value stays at ``1.0``. + + *appearance* + Whether the window is rendered in dark mode on Windows and macOS: + ``'auto'``, ``'light'`` or ``'dark'`` (this has no effect on X11). + + *fullscreen* + Whether the window takes up the entire screen and has no borders. + + *topmost* + Whether the window is displayed above all other windows. + + Windows additionally supports: + + *disabled* + Whether the window is in a disabled state. + + *toolwindow* + Whether the window uses the tool window style. + + *transparentcolor* + The color that is made fully transparent, or an empty string for none. + + macOS additionally supports: + + *class* + Whether the underlying Aqua window is an ``nswindow`` or an + ``nspanel``; this can only be set before the window is created. + + *modified* + The modification state shown by the window's close button and proxy + icon. + + *notify* + Whether the application's dock icon bounces to request attention. + + *stylemask* + The style mask of the underlying Aqua window, given as a list of bit + names such as ``titled`` or ``resizable``. + + *tabbingid* + The identifier of the tab group that the window belongs to. + + *tabbingmode* + Whether the window may be opened as a tab: ``'auto'``, ``'preferred'`` + or ``'disallowed'``. + + *titlepath* + The path of the file represented by the window's proxy icon. + + *transparent* + Whether the content area is transparent and the window shadow is + turned off. + + X11 additionally supports: + + *type* + The window type, or a list of types in order of preference, that the + window manager should use to interpret the window, such as + ``'dialog'`` or ``'splash'``. + + *zoomed* + Whether the window is maximized. + + .. note:: + + Tk 8.6 added the *type* attribute, and Tk 9.0 added the *appearance*, + *class*, *stylemask*, *tabbingid* and *tabbingmode* attributes. + + On X11 changes are applied asynchronously, so a queried value may not yet + reflect the most recent request. + :meth:`wm_attributes` is an alias of :meth:`!attributes`. + + .. versionchanged:: 3.13 + A single attribute may now be queried by name without the leading + ``-``, and attributes may be set using keyword arguments. + The *return_python_dict* parameter was added. + + .. deprecated:: 3.13 + Setting an attribute by passing the option name (with a leading + ``-``) and its value as two positional arguments, as in + ``w.attributes('-alpha', 0.5)``, is deprecated; use keyword arguments + instead. + + + .. method:: wm_client(name=None) + :no-typesetting: + + .. method:: client(name=None) + + Store *name*, which should be the name of the host on which the + application is running, in the window's ``WM_CLIENT_MACHINE`` property + for use by the window or session manager. + An empty string deletes the property. + With no argument, return the last name set, or an empty string. + :meth:`wm_client` is an alias of :meth:`!client`. + + .. method:: wm_colormapwindows(*wlist) + :no-typesetting: + + .. method:: colormapwindows(*wlist) + + Manipulate the ``WM_COLORMAP_WINDOWS`` property, which tells the window + manager about windows that have private colormaps. + If *wlist* is given, overwrite the property with those windows (their + order is a priority order for installing colormaps). + With no arguments, return the list of windows currently named in the + property. + :meth:`wm_colormapwindows` is an alias of :meth:`!colormapwindows`. + + .. method:: wm_command(value=None) + :no-typesetting: + + .. method:: command(value=None) + + Store *value* in the window's ``WM_COMMAND`` property for use by the + window or session manager; it should be a list giving the words of the + command used to invoke the application. + An empty string deletes the property. + With no argument, return the last value set, or an empty string. + :meth:`wm_command` is an alias of :meth:`!command`. + + .. method:: wm_deiconify() + :no-typesetting: + + .. method:: deiconify() + + Display the window in normal (non-iconified) form by mapping it. + If the window has never been mapped, this ensures it appears de-iconified + when it is first mapped. + On Windows the window is also raised and given the focus. + :meth:`wm_deiconify` is an alias of :meth:`!deiconify`. + + .. method:: wm_focusmodel(model=None) + :no-typesetting: + + .. method:: focusmodel(model=None) + + Set or query the focus model for the window. + *model* is either ``'active'`` (the window claims the input focus for + itself or its descendants, even when the focus is in another application) + or ``'passive'`` (the window relies on the window manager to give it the + focus). + With no argument, return the current model. + The default is ``'passive'``, which is what the :meth:`!focus` command + assumes. + :meth:`wm_focusmodel` is an alias of :meth:`!focusmodel`. + + .. method:: wm_forget(window) + :no-typesetting: + + .. method:: forget(window) + + Unmap *window* from the screen so that it is no longer managed by the + window manager. + A :class:`Toplevel` is then treated like a :class:`Frame`, although its + ``-menu`` configuration is remembered and the menu reappears if the + widget is managed again. + :meth:`wm_forget` is an alias of :meth:`!forget`. + + .. versionadded:: 3.3 + + .. method:: wm_frame() + :no-typesetting: + + .. method:: frame() + + Return the platform-specific window identifier for the outermost + decorative frame containing the window, if the window manager has + reparented it into such a frame; otherwise return the identifier of the + window itself. + :meth:`wm_frame` is an alias of :meth:`!frame`. + + .. method:: wm_geometry(newGeometry=None) + :no-typesetting: + + .. method:: geometry(newGeometry=None) + + Set or query the geometry of the window. + *newGeometry* has the form ``=widthxheight+x+y``, where any of ``=``, + ``widthxheight`` and the ``+x+y`` position may be omitted. + *width* and *height* are in pixels (or grid units for a gridded window); + a position preceded by ``+`` is measured from the left or top edge of the + screen and one preceded by ``-`` from the right or bottom edge. + An empty string cancels any user-specified geometry, letting the window + revert to its natural size. + With no argument, return the current geometry as a string of the form + ``'200x200+10+10'``. + :meth:`wm_geometry` is an alias of :meth:`!geometry`. + + .. method:: wm_grid(baseWidth=None, baseHeight=None, widthInc=None, heightInc=None) + :no-typesetting: + + .. method:: grid(baseWidth=None, baseHeight=None, widthInc=None, heightInc=None) + + Manage the window as a gridded window and define the relationship between + grid units and pixels. + *baseWidth* and *baseHeight* are the numbers of grid units for the + window's internally requested size, and *widthInc* and *heightInc* are + the pixel sizes of a horizontal and vertical grid unit. + Empty strings turn off gridded management. + With no arguments, return a tuple of the four current values, or an empty + string if the window is not gridded. + :meth:`wm_grid` is an alias of :meth:`!grid`. + + .. method:: wm_group(pathName=None) + :no-typesetting: + + .. method:: group(pathName=None) + + Set or query the leader of a group of related windows. + *pathName* gives the path name of the group leader; the window manager + may, for example, unmap all windows in the group when the leader is + iconified. + An empty string removes the window from any group. + With no argument, return the path name of the current group leader, or an + empty string. + :meth:`wm_group` is an alias of :meth:`!group`. + + .. method:: wm_iconbitmap(bitmap=None, default=None) + :no-typesetting: + + .. method:: iconbitmap(bitmap=None, default=None) + + Set or query the bitmap used by the window manager for the window's icon. + *bitmap* names a bitmap in one of the standard forms accepted by Tk; an + empty string cancels the current icon bitmap. + With no argument, return the name of the current icon bitmap, or an empty + string. + On Windows the *default* argument names an icon (for example an ``.ico`` + file) applied to all top-level windows that have no icon of their own. + :meth:`wm_iconbitmap` is an alias of :meth:`!iconbitmap`. + + .. method:: wm_iconify() + :no-typesetting: + + .. method:: iconify() + + Iconify the window. + If the window has not yet been mapped for the first time, arrange for it + to appear in the iconified state when it is eventually mapped. + :meth:`wm_iconify` is an alias of :meth:`!iconify`. + + .. method:: wm_iconmask(bitmap=None) + :no-typesetting: + + .. method:: iconmask(bitmap=None) + + Set or query the bitmap used as a mask for the icon (see + :meth:`iconbitmap`). + Where the mask is zero no icon is displayed; where it is one, the + corresponding bits of the icon bitmap are shown. + An empty string cancels the current mask. + With no argument, return the name of the current icon mask, or an empty + string. + :meth:`wm_iconmask` is an alias of :meth:`!iconmask`. + + .. method:: wm_iconname(newName=None) + :no-typesetting: + + .. method:: iconname(newName=None) + + Set or query the name displayed by the window manager inside the window's + icon. + With no argument, return the current icon name, or an empty string if + none has been set (in which case the window manager normally displays the + window's title). + :meth:`wm_iconname` is an alias of :meth:`!iconname`. + + .. method:: wm_iconphoto(default=False, *images) + :no-typesetting: + + .. method:: iconphoto(default=False, *images) + + Set the titlebar icon for the window from one or more :class:`PhotoImage` + objects given in *images*. + Several images of different sizes (for example 16x16 and 32x32) may be + supplied so that the window manager can choose an appropriate one. + The image data is taken as a snapshot at the time of the call; later + changes to the images are not reflected. + If *default* is true, the icon is also applied to all top-level windows + created in the future. + On macOS only the first image is used. + :meth:`wm_iconphoto` is an alias of :meth:`!iconphoto`. + + .. versionadded:: 3.3 + + .. method:: wm_iconposition(x=None, y=None) + :no-typesetting: + + .. method:: iconposition(x=None, y=None) + + Set or query a hint to the window manager about where the window's icon + should be positioned. + Empty strings cancel an existing hint. + With no arguments, return a tuple of the two current values, or an empty + string if no hint is in effect. + :meth:`wm_iconposition` is an alias of :meth:`!iconposition`. + + .. method:: wm_iconwindow(pathName=None) + :no-typesetting: + + .. method:: iconwindow(pathName=None) + + Set or query the window used as the icon for the window. + When the window is iconified, *pathName* is mapped to serve as its icon + and unmapped again when it is de-iconified. + An empty string cancels the association. + With no argument, return the path name of the current icon window, or an + empty string. + Not all window managers support icon windows, and the concept is + meaningless on non-X11 platforms. + :meth:`wm_iconwindow` is an alias of :meth:`!iconwindow`. + + .. method:: wm_manage(widget) + :no-typesetting: + + .. method:: manage(widget) + + Make *widget* a stand-alone top-level window, decorated by the window + manager with a title bar and so on. + Only :class:`Frame`, :class:`LabelFrame` and :class:`Toplevel` widgets + may be used; passing any other widget type raises an error. + :meth:`wm_manage` is an alias of :meth:`!manage`. + + .. versionadded:: 3.3 + + .. method:: wm_maxsize(width=None, height=None) + :no-typesetting: + + .. method:: maxsize(width=None, height=None) + + Set or query the maximum permissible dimensions of the window, in pixels + (or grid units for a gridded window). + The window manager restricts the window to be no larger than *width* and + *height*. + With no arguments, return a tuple of the current maximum width and + height. + The maximum size defaults to the size of the screen. + :meth:`wm_maxsize` is an alias of :meth:`!maxsize`. + + .. method:: wm_minsize(width=None, height=None) + :no-typesetting: + + .. method:: minsize(width=None, height=None) + + Set or query the minimum permissible dimensions of the window, in pixels + (or grid units for a gridded window). + The window manager restricts the window to be no smaller than *width* and + *height*. + With no arguments, return a tuple of the current minimum width and + height. + The minimum size defaults to one pixel in each dimension. + :meth:`wm_minsize` is an alias of :meth:`!minsize`. + + .. method:: wm_overrideredirect(boolean=None) + :no-typesetting: + + .. method:: overrideredirect(boolean=None) + + Set or query the override-redirect flag for the window. + When this flag is set, the window is ignored by the window manager: it is + not reparented into a decorative frame and the user cannot manipulate it + through the usual window manager controls. + With no argument, return a boolean indicating whether the flag is set. + The flag is reliably honored only when the window is first mapped or + remapped from the withdrawn state. + :meth:`wm_overrideredirect` is an alias of :meth:`!overrideredirect`. + + .. method:: wm_positionfrom(who=None) + :no-typesetting: + + .. method:: positionfrom(who=None) + + Set or query the source of the window's current position. + *who* is either ``'program'`` or ``'user'`` and indicates whether the + position was requested by the program or by the user; an empty string + cancels the current source. + With no argument, return the current source, or an empty string if none + has been set. + Tk automatically sets the source to ``'user'`` when :meth:`geometry` is + called, unless it has been set explicitly to ``'program'``. + :meth:`wm_positionfrom` is an alias of :meth:`!positionfrom`. + + .. method:: wm_protocol(name=None, func=None) + :no-typesetting: + + .. method:: protocol(name=None, func=None) + + Register *func* as the handler for the window manager protocol *name*, an + atom such as ``'WM_DELETE_WINDOW'``, ``'WM_SAVE_YOURSELF'`` or + ``'WM_TAKE_FOCUS'``; *func* is then called whenever the window manager + sends a message of that protocol. + Tk installs a default ``WM_DELETE_WINDOW`` handler that destroys the + window, which this method can replace. + If *func* is an empty string, the handler is removed. + With only *name*, return the name of its registered handler command, or + an empty string if none is set (the default ``WM_DELETE_WINDOW`` handler + is not reported); with no arguments, return a tuple of the protocols that + currently have handlers. + :meth:`wm_protocol` is an alias of :meth:`!protocol`. + + .. method:: wm_resizable(width=None, height=None) + :no-typesetting: + + .. method:: resizable(width=None, height=None) + + Control whether the user may interactively resize the window. + *width* and *height* are boolean values that determine whether the + window's width and height may be changed. + With no arguments, return a tuple of two ``0``/``1`` values indicating + whether each dimension is currently resizable. + By default a window is resizable in both dimensions. + :meth:`wm_resizable` is an alias of :meth:`!resizable`. + + .. method:: wm_sizefrom(who=None) + :no-typesetting: + + .. method:: sizefrom(who=None) + + Set or query the source of the window's current size. + *who* is either ``'program'`` or ``'user'`` and indicates whether the + size was requested by the program or by the user; an empty string cancels + the current source. + With no argument, return the current source, or an empty string if none + has been set. + :meth:`wm_sizefrom` is an alias of :meth:`!sizefrom`. + + .. method:: wm_state(newstate=None) + :no-typesetting: + + .. method:: state(newstate=None) + + Set or query the state of the window. + With no argument, return the current state: one of ``'normal'``, + ``'iconic'``, ``'withdrawn'``, ``'icon'`` or, on Windows and macOS only, + ``'zoomed'``. + ``'iconic'`` refers to a window that has been iconified, while ``'icon'`` + refers to a window serving as the icon for another window (see + :meth:`iconwindow`); the ``'icon'`` state cannot be set. + :meth:`wm_state` is an alias of :meth:`!state`. + + .. method:: wm_title(string=None) + :no-typesetting: + + .. method:: title(string=None) + + Set or query the title for the window, which the window manager should + display in the window's title bar. + With no argument, return the current title. + The title defaults to the window's name. + :meth:`wm_title` is an alias of :meth:`!title`. + + .. method:: wm_transient(master=None) + :no-typesetting: + + .. method:: transient(master=None) + + Mark the window as a transient window (such as a pull-down menu or + dialog) working on behalf of *master*, the path name of another top-level + window. + An empty string clears the transient status. + With no argument, return the path name of the current master, or an empty + string. + A transient window mirrors state changes in its master and may be + decorated differently by the window manager; it is an error to make a + window a transient of itself. + :meth:`wm_transient` is an alias of :meth:`!transient`. + + .. method:: wm_withdraw() + :no-typesetting: + + .. method:: withdraw() + + Withdraw the window from the screen, unmapping it and causing the window + manager to forget about it. + If the window has never been mapped, it is instead mapped in the + withdrawn state. + It is sometimes necessary to withdraw a window and then re-map it (for + example with :meth:`deiconify`) to make some window managers notice + changes to window attributes. + :meth:`wm_withdraw` is an alias of :meth:`!withdraw`. + + +.. class:: Pack() + + Geometry manager that arranges widgets by packing them against the sides of + their container. + The :class:`!Pack` mix-in is inherited by all widgets (through + :class:`Widget`) and provides the methods for managing a widget with the + *pack* geometry manager. + See also :ref:`pack-the-packer`. + + .. method:: configure(cnf={}, **kw) + :no-typesetting: + + .. method:: config(cnf={}, **kw) + :no-typesetting: + + .. method:: pack_configure(cnf={}, **kw) + pack(cnf={}, **kw) + + Pack the widget inside its container, positioning it relative to the + siblings already packed there. + The supported options are: + + *side* + Which side of the container to pack the widget against: ``'top'`` (the + default), ``'bottom'``, ``'left'`` or ``'right'``. + + *fill* + Whether to stretch the widget to fill its parcel: ``'none'`` (the + default), ``'x'``, ``'y'`` or ``'both'``. + + *expand* + Whether the widget should expand to consume any extra space in its + container (a boolean, default false). + + *anchor* + Where to position the widget in its parcel when the parcel is larger + than the widget: an anchor such as ``'n'`` or ``'sw'`` (default + ``'center'``). + + *ipadx*, *ipady* + Internal padding added on the left and right (*ipadx*) or top and + bottom (*ipady*) of the widget, as a screen distance (default ``0``). + + *padx*, *pady* + External padding left on the left and right (*padx*) or top and bottom + (*pady*) of the widget, as a screen distance or a pair of two + distances for the two sides (default ``0``). + + *after* + Pack the widget after the given widget in the packing order, using the + same container. + + *before* + Pack the widget before the given widget in the packing order, using + the same container. + + *in_* + The container in which to pack the widget; it defaults to the parent + widget. + + :meth:`pack`, :meth:`configure` and :meth:`config` are aliases of + :meth:`!pack_configure`. + + .. method:: forget() + :no-typesetting: + + .. method:: pack_forget() + + Unmap the widget and remove it from the packing order, forgetting its + packing options. + It can be packed again later with :meth:`pack_configure`. + :meth:`forget` is an alias of :meth:`!pack_forget`. + + .. method:: info() + :no-typesetting: + + .. method:: pack_info() + + Return a dictionary of the widget's current packing options. + :meth:`info` is an alias of :meth:`!pack_info`. + + .. method:: propagate() + propagate(flag) + :no-typesetting: + + .. method:: pack_propagate() + pack_propagate(flag) + + Same as :meth:`Misc.pack_propagate`, treating this widget as a container: + enable or disable geometry propagation. + :meth:`propagate` is an alias of :meth:`!pack_propagate`. + + .. method:: slaves() + :no-typesetting: + + .. method:: pack_slaves() + + Same as :meth:`Misc.pack_slaves`: return the list of widgets packed in + this widget. + :meth:`slaves` is an alias of :meth:`!pack_slaves`. + + .. method:: content() + :no-typesetting: + + .. method:: pack_content() + + Same as :meth:`Misc.pack_content`. + :meth:`content` is an alias of :meth:`!pack_content`. + + .. versionadded:: 3.15 + + +.. class:: Place() + + Geometry manager that places widgets at explicit positions and sizes within + their container. + The :class:`!Place` mix-in is inherited by all widgets (through + :class:`Widget`). + + .. method:: configure(cnf={}, **kw) + :no-typesetting: + + .. method:: config(cnf={}, **kw) + :no-typesetting: + + .. method:: place_configure(cnf={}, **kw) + place(cnf={}, **kw) + + Place the widget inside its container at an absolute or relative + position. + The supported options are: + + *x*, *y* + The absolute horizontal and vertical position of the widget's anchor + point, as a screen distance (default ``0``). + + *relx*, *rely* + The horizontal and vertical position of the widget's anchor point as a + fraction of the container's width and height, where ``0.0`` is the + left or top edge and ``1.0`` is the right or bottom edge. + If both the absolute and the relative option are given, their values + are summed. + + *anchor* + Which point of the widget is placed at the given position: an anchor + such as ``'n'`` or ``'se'`` (default ``'nw'``). + + *width*, *height* + The absolute width and height of the widget, as a screen distance. + By default the widget's requested size is used. + + *relwidth*, *relheight* + The width and height of the widget as a fraction of the container's + width and height. + If both the absolute and the relative option are given, their values + are summed. + + *bordermode* + How the container's border affects placement: ``'inside'`` (the + default) measures the area inside the border, ``'outside'`` measures + the area including the border, and ``'ignore'`` uses the official X + area. + + *in_* + The container relative to which the widget is placed; it must be the + widget's parent or a descendant of the parent, and defaults to the + parent. + + :meth:`place`, :meth:`configure` and :meth:`config` are aliases of + :meth:`!place_configure`. + + .. method:: forget() + :no-typesetting: + + .. method:: place_forget() + + Unmap the widget and remove it from the placement, forgetting its place + options. + :meth:`forget` is an alias of :meth:`!place_forget`. + + .. method:: info() + :no-typesetting: + + .. method:: place_info() + + Return a dictionary of the widget's current place options. + :meth:`info` is an alias of :meth:`!place_info`. + + .. method:: slaves() + :no-typesetting: + + .. method:: place_slaves() + + Same as :meth:`Misc.place_slaves`: return the list of widgets placed in + this widget. + :meth:`slaves` is an alias of :meth:`!place_slaves`. + + .. method:: content() + :no-typesetting: + + .. method:: place_content() + + Same as :meth:`Misc.place_content`. + :meth:`content` is an alias of :meth:`!place_content`. + + .. versionadded:: 3.15 + + +.. class:: Grid() + + Geometry manager that arranges widgets in a two-dimensional grid of rows and + columns within their container. + The :class:`!Grid` mix-in is inherited by all widgets (through + :class:`Widget`). + + .. method:: configure(cnf={}, **kw) + :no-typesetting: + + .. method:: config(cnf={}, **kw) + :no-typesetting: + + .. method:: grid_configure(cnf={}, **kw) + grid(cnf={}, **kw) + + Position the widget in a cell of its container's grid. + The supported options are: + + *row*, *column* + The row and column of the cell to place the widget in, counting from + ``0``. + *column* defaults to the column after the previous widget placed in + the same :meth:`!grid_configure` call (or ``0``), and *row* defaults + to the next empty row. + + *rowspan*, *columnspan* + The number of rows and columns the widget should span (default ``1``). + + *sticky* + How to position or stretch the widget when its cell is larger than the + widget: a string containing zero or more of the characters ``'n'``, + ``'s'``, ``'e'`` and ``'w'``, naming the cell sides the widget sticks + to. + Specifying both ``'n'`` and ``'s'`` (or ``'e'`` and ``'w'``) stretches + the widget to fill the height (or width) of the cell. + The default is ``''``, which centers the widget at its requested size. + + *ipadx*, *ipady* + Internal padding added on the left and right (*ipadx*) or top and + bottom (*ipady*) of the widget, as a screen distance (default ``0``). + + *padx*, *pady* + External padding left on the left and right (*padx*) or top and bottom + (*pady*) of the widget, as a screen distance or a pair of two + distances for the two sides (default ``0``). + + *in_* + The container in whose grid to place the widget; it defaults to the + parent widget. + + :meth:`grid`, :meth:`configure` and :meth:`config` are aliases of + :meth:`!grid_configure`. + + .. method:: forget() + :no-typesetting: + + .. method:: grid_forget() + + Unmap the widget and remove it from the grid, forgetting its grid + options. + :meth:`forget` is an alias of :meth:`!grid_forget`. + + .. method:: grid_remove() + + Unmap the widget and remove it from the grid, but remember its grid + options so that it is restored to the same cell if it is gridded again. + + .. method:: info() + :no-typesetting: + + .. method:: grid_info() + + Return a dictionary of the widget's current grid options. + :meth:`info` is an alias of :meth:`!grid_info`. + + .. method:: bbox(column=None, row=None, col2=None, row2=None) + :no-typesetting: + + .. method:: grid_bbox(column=None, row=None, col2=None, row2=None) + + Same as :meth:`Misc.grid_bbox`. + :meth:`bbox` is an alias of :meth:`!grid_bbox`. + + .. method:: columnconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_columnconfigure(index, cnf={}, **kw) + + Same as :meth:`Misc.grid_columnconfigure`: query or set the options (such + as *weight*, *minsize*, *pad* and *uniform*) of a grid column. + :meth:`columnconfigure` is an alias of :meth:`!grid_columnconfigure`. + + .. method:: rowconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_rowconfigure(index, cnf={}, **kw) + + Same as :meth:`Misc.grid_rowconfigure`: query or set the options of a + grid row. + :meth:`rowconfigure` is an alias of :meth:`!grid_rowconfigure`. + + .. method:: location(x, y) + :no-typesetting: + + .. method:: grid_location(x, y) + + Same as :meth:`Misc.grid_location`: return the ``(column, row)`` of the + cell that covers the pixel at *x*, *y*. + :meth:`location` is an alias of :meth:`!grid_location`. + + .. method:: size() + :no-typesetting: + + .. method:: grid_size() + + Same as :meth:`Misc.grid_size`: return a ``(columns, rows)`` tuple giving + the size of the grid. + :meth:`size` is an alias of :meth:`!grid_size`. + + .. method:: propagate() + propagate(flag) + :no-typesetting: + + .. method:: grid_propagate() + grid_propagate(flag) + + Same as :meth:`Misc.grid_propagate`. + :meth:`propagate` is an alias of :meth:`!grid_propagate`. + + .. method:: slaves(row=None, column=None) + :no-typesetting: + + .. method:: grid_slaves(row=None, column=None) + + Same as :meth:`Misc.grid_slaves`: return the widgets managed in the grid, + optionally restricted to a *row* and/or *column*. + :meth:`slaves` is an alias of :meth:`!grid_slaves`. + + .. method:: content(row=None, column=None) + :no-typesetting: + + .. method:: grid_content(row=None, column=None) + + Same as :meth:`Misc.grid_content`. + :meth:`content` is an alias of :meth:`!grid_content`. + + .. versionadded:: 3.15 + + +.. class:: XView() + + Mix-in providing the horizontal-scrolling interface shared by widgets such + as :class:`Entry`, :class:`Canvas`, :class:`Listbox`, :class:`Text` and + :class:`Spinbox`. + A widget's :meth:`xview` method is registered as the *command* of a + horizontal :class:`Scrollbar`. + + .. method:: xview(*args) + + Query or change the horizontal position of the view. + With no arguments, return a tuple ``(first, last)`` of two fractions + between 0 and 1 giving the portion of the document that is currently + visible. + Otherwise the arguments are passed to the Tk ``xview`` widget command and + are usually generated by a scrollbar; :meth:`xview_moveto` and + :meth:`xview_scroll` provide a more convenient interface. + + .. method:: xview_moveto(fraction) + + Adjust the view so that *fraction* of the total width of the document is + off-screen to the left. + *fraction* is a number between 0 and 1. + + .. method:: xview_scroll(number, what) + + Shift the view left or right by *number* units. + *what* is either ``'units'`` or ``'pages'``; a negative *number* scrolls + left and a positive one scrolls right. + + +.. class:: YView() + + Mix-in providing the vertical-scrolling interface shared by widgets such as + :class:`Canvas`, :class:`Listbox` and :class:`Text`. + A widget's :meth:`yview` method is registered as the *command* of a vertical + :class:`Scrollbar`. + + .. method:: yview(*args) + + Query or change the vertical position of the view. + With no arguments, return a tuple ``(first, last)`` of two fractions + between 0 and 1 giving the portion of the document that is currently + visible. + Otherwise the arguments are passed to the Tk ``yview`` widget command, + usually generated by a scrollbar; :meth:`yview_moveto` and + :meth:`yview_scroll` provide a more convenient interface. + + .. method:: yview_moveto(fraction) + + Adjust the view so that *fraction* of the total height of the document is + off-screen above the top. + *fraction* is a number between 0 and 1. + + .. method:: yview_scroll(number, what) + + Shift the view up or down by *number* units. + *what* is either ``'units'`` or ``'pages'``; a negative *number* scrolls + up and a positive one scrolls down. + + +.. class:: BaseWidget(master, widgetName, cnf={}, kw={}, extra=()) + + Internal base class for all widgets. + It inherits from :class:`Misc` and adds the machinery that creates the + underlying Tk widget; application code normally uses :class:`Widget` or a + concrete widget class rather than instantiating :class:`!BaseWidget` + directly. + + .. method:: destroy() + + Destroy this widget and all of its children, removing the corresponding + Tk widgets and deleting the associated Tcl commands. + + +.. class:: Widget(master, widgetName, cnf={}, kw={}, extra=()) + + Internal base class for the standard widgets. + It combines :class:`BaseWidget` with the geometry-manager mix-ins + :class:`Pack`, :class:`Place` and :class:`Grid`, so that every widget can be + managed by any of the three geometry managers. + The concrete widget classes (:class:`Button`, :class:`Label`, and so on) + derive from :class:`!Widget`. + + +.. class:: Toplevel(master=None, cnf={}, **kw) + + A :class:`!Toplevel` widget is a top-level window, similar to a + :class:`Frame` except that its X parent is the root window of a screen + rather than its logical parent. + Its primary purpose is to serve as a container for dialog boxes and other + collections of widgets; its only visible features are its background and an + optional 3-D border. + Notable options include *menu*, which installs a :class:`Menu` as the + window's menubar. + Inherits from :class:`BaseWidget` and :class:`Wm`, so a toplevel is managed + by the window manager. + Refer to the Tk ``toplevel`` manual page for the full list of options. + + +Widget classes +^^^^^^^^^^^^^^ + +.. class:: Button(master=None, cnf={}, **kw) + + A :class:`!Button` widget displays a textual string, bitmap or image and + invokes a command when the user presses it (by clicking mouse button 1 over + the button or, when the button has focus, by pressing the space key). + Inherits from :class:`Widget`. + In addition to the standard widget options, a button accepts the options + documented in the Tk ``button`` manual page, such as *command* (the callback + invoked when the button is pressed), *textvariable*, *state* and *default*. + + .. method:: invoke() + + Invoke the command associated with the button, if there is one, and + return its result, or an empty string if no command is associated with + the button. + This is ignored if the button's state is ``disabled``. + + .. method:: flash() + + Flash the button by redisplaying it several times, alternating between + the active and normal colors. + At the end of the flash the button is left in the same normal or active + state as when the method was called. + This is ignored if the button's state is ``disabled``. + + +.. class:: Canvas(master=None, cnf={}, **kw) + + A :class:`!Canvas` widget implements structured graphics. + It displays any number of *items*, such as arcs, lines, ovals, polygons, + rectangles, text, bitmaps, images and embedded windows, which may be drawn, + moved, re-colored and bound to events. + Inherits from :class:`Widget`, :class:`XView` and :class:`YView`, so the + view can be scrolled horizontally and vertically with :meth:`~XView.xview` + and :meth:`~YView.yview`. + Refer to the Tk ``canvas`` manual page for the full list of widget and item + options. + + Each item has a unique integer *id*, assigned when it is created, and zero + or more string *tags*. + A tag is an arbitrary string that does not have the form of an integer; the + same tag may be shared by many items, which makes tags convenient for + grouping items. + The special tag ``'all'`` matches every item in the canvas, and + ``'current'`` matches the topmost item under the mouse pointer. + Most methods take a *tagOrId* argument that may be an integer id naming a + single item, or a tag naming zero or more items; as described in the Tk + ``canvas`` manual page, a tag may also be a logical expression of tags + combined with the operators ``&&``, ``||``, ``^``, ``!`` and parentheses. + When a method that operates on a single item is given a *tagOrId* matching + several items, it normally uses the lowest matching item in the display + list. + + The items are kept in a *display list* that determines drawing order: items + later in the list are drawn on top of earlier ones. + A newly created item is placed at the top of the list; the order can be + changed with :meth:`tag_raise` and :meth:`tag_lower`. + + .. method:: create_arc(*args, **kw) + create_bitmap(*args, **kw) + create_image(*args, **kw) + create_line(*args, **kw) + create_oval(*args, **kw) + create_polygon(*args, **kw) + create_rectangle(*args, **kw) + create_text(*args, **kw) + create_window(*args, **kw) + + Create a new item of the corresponding type and return its integer id. + Each method is called as ``create_TYPE(coord..., **options)``: the + leading positional arguments give the coordinates that define the item + (as separate numbers, as a single sequence of numbers, or as coordinate + pairs), and the keyword arguments set item-specific options. + Coordinates and screen distances may be given as numbers (interpreted as + pixels) or as strings with a unit suffix (``'m'``, ``'c'``, ``'i'`` or + ``'p'`` for millimetres, centimetres, inches or printer's points), but + are always stored and returned in pixels. + + The item types are: ``arc`` (an arc-shaped region that is a section of an + oval, defined by two diagonally opposite corners ``x1, y1, x2, y2`` of + the enclosing rectangle); ``bitmap`` (a two-color bitmap positioned at a + point ``x, y``); ``image`` (a Tk image positioned at a point ``x, y``); + ``line`` (a line or curve through the points ``x1, y1, ..., xn, yn``); + ``oval`` (a circle or ellipse inscribed in the rectangle + ``x1, y1, x2, y2``); ``polygon`` (a closed polygon through the points + ``x1, y1, ..., xn, yn``); ``rectangle`` (a rectangle with corners + ``x1, y1, x2, y2``); ``text`` (a string of text positioned at a point + ``x, y``); and ``window`` (a child widget embedded in the canvas at a + point ``x, y``, specified with the *window* option). + + Most item types accept a common set of *standard item options*, plus a + few options specific to each type. + Option names are passed as keyword arguments, without the leading + hyphen. + + The standard item options are: + + *fill* + The color used to fill the item's interior, or to draw a *line* + item or the characters of a *text* item. + An empty string (the default for all types except *line* and *text*) + leaves the item unfilled. + + *outline* + The color used to draw the item's outline. + An empty string draws no outline. + + *width* + The width of the outline, defaulting to ``1.0``. + Has no effect if *outline* is empty. + + *dash* + A dash pattern for the outline, given either as a sequence of + segment lengths in pixels or as a string of the characters + ``'.'``, ``','``, ``'-'``, ``'_'`` and space. + An empty pattern (the default) draws a solid outline. + + *dashoffset* + The starting offset in pixels into the *dash* pattern. + Ignored if there is no *dash* pattern. + + *stipple* + A bitmap used as a stipple pattern when filling the item. + Only well supported on X11. + + *outlinestipple* + A bitmap used as a stipple pattern when drawing the outline. + Has no effect if *outline* is empty. + + *offset*, *outlineoffset* + The offset of the fill and outline stipple patterns, given as + ``'x,y'`` or as a side such as ``'n'``, ``'se'`` or ``'center'``. + Stipple offsets are only supported on X11. + + *state* + Overrides the canvas state for this item; one of ``'normal'``, + ``'disabled'`` or ``'hidden'``. + + *tags* + A single tag or a sequence of tags to associate with the item, + replacing any existing tags. + + Many of these options have *active...* and *disabled...* variants + (such as *activefill*, *disabledfill*, *activewidth*, *disableddash*, + *activeoutline*, *disabledstipple*) that override the base option when + the item is the active item (under the mouse pointer) or is in the + disabled state. + + The following item types support additional options. + + For ``arc`` items: + + *start* + The start of the arc's angular range, in degrees measured + counter-clockwise from the 3-o'clock position. + + *extent* + The size of the angular range, in degrees counter-clockwise from + *start*. + + *style* + How the arc is drawn: ``'pieslice'`` (the default), ``'chord'`` or + ``'arc'``. + + For ``line`` items: + + *arrow* + Where to draw arrowheads: ``'none'`` (the default), ``'first'``, + ``'last'`` or ``'both'``. + + *arrowshape* + A sequence of three distances describing the shape of the + arrowheads. + + *capstyle* + How line ends are drawn: ``'butt'`` (the default), ``'projecting'`` + or ``'round'``. + + *joinstyle* + How line vertices are drawn: ``'round'`` (the default), ``'bevel'`` + or ``'miter'``. + + *smooth* + The smoothing method: a false value (the default) for no smoothing, + or ``'true'``/``'bezier'`` or ``'raw'`` to draw the line as a curve. + + *splinesteps* + The number of line segments approximating each spline when + *smooth* is enabled. + + For ``polygon`` items: + + *joinstyle*, *smooth*, *splinesteps* + As for ``line`` items, applied to the polygon's outline. + + For ``text`` items: + + *text* + The string to display; newline characters start new lines. + + *font* + The font used for the text. + + *justify* + How lines are justified: ``'left'`` (the default), ``'right'`` or + ``'center'``. + + *anchor* + How the text is positioned relative to its point, defaulting to + ``'center'``. + + *width* + The maximum line length; if non-zero, lines are wrapped at spaces. + + *angle* + How many degrees to rotate the text counter-clockwise about its + positioning point, from ``0.0`` to ``360.0`` (default ``0.0``). + + *underline* + The index of a character to underline, or ``-1`` for none. + + For ``bitmap`` items: + + *bitmap* + The bitmap to display. + + *anchor* + How the bitmap is positioned relative to its point. + + *background*, *foreground* + The colors used for the bitmap's ``0`` and ``1`` pixels; an empty + *background* makes the ``0`` pixels transparent. + Both have *active...* and *disabled...* variants, and *bitmap* has + *activebitmap* and *disabledbitmap* variants. + + For ``image`` items: + + *image* + The Tk image to display, previously created with the image + protocols. + + *anchor* + How the image is positioned relative to its point. + + Both options have *active...* and *disabled...* variants + (*activeimage*, *disabledimage*) used in the active and disabled + states. + + For ``window`` items: + + *window* + The widget to embed; it must be a child of the canvas or of one of + its ancestors, and may not be a top-level window. + + *anchor* + How the window is positioned relative to its point. + + *width*, *height* + The size to assign to the window; if zero (the default), the window + is given its requested size. + + ``oval`` and ``rectangle`` items have no type-specific options; they + use only the standard item options. + + .. note:: + + Tk 8.6 added the *angle* option and Tk 9.0 added the *underline* + option for ``text`` items. + + .. method:: coords(tagOrId) + coords(tagOrId, coordList, /) + coords(tagOrId, /, *coordList) + + Query or modify the coordinates of an item. + With only *tagOrId*, return a list of the floating-point coordinates of + the item given by *tagOrId* (the first matching item if it matches + several). + Given new coordinates, replace the coordinates of that item with them; + like the ``create_*`` methods, the coordinates may be given as separate + numbers, as a single sequence, or as coordinate pairs. + The returned coordinates are always in pixels, regardless of the units + used to specify them; for rectangles, ovals and arcs they are ordered + left, top, right, bottom. + + .. versionchanged:: 3.12 + The arguments are now flattened: the coordinates may be given as + separate arguments, as a single sequence, or grouped in pairs, like + the ``create_*`` methods. + + + .. method:: move(tagOrId, xAmount, yAmount, /) + + Move each of the items given by *tagOrId* in the canvas coordinate space + by adding *xAmount* to every x-coordinate and *yAmount* to every + y-coordinate of the item. + + .. method:: moveto(tagOrId, x='', y='') + + Move the items given by *tagOrId* so that the first coordinate pair (the + upper-left corner of the bounding box) of the lowest matching item is at + position (*x*, *y*). + *x* or *y* may be an empty string, in which case the corresponding + coordinate is unchanged. + All matching items keep their positions relative to each other. + + .. versionadded:: 3.8 + + + .. method:: scale(tagOrId, xOrigin, yOrigin, xScale, yScale, /) + + Rescale the coordinates of all items given by *tagOrId* in canvas + coordinate space. + Each x-coordinate is adjusted so that its distance from *xOrigin* changes + by a factor of *xScale*, and each y-coordinate so that its distance from + *yOrigin* changes by a factor of *yScale* (a factor of ``1.0`` leaves the + coordinate unchanged). + + .. method:: delete(*tagOrIds) + + Delete each of the items given by the *tagOrIds* arguments. + + .. method:: dchars(tagOrId, first, /) + dchars(tagOrId, first, last, /) + + Delete from each of the items given by *tagOrId* the characters (for text + items) or coordinates (for line and polygon items) in the range from + *first* to *last* inclusive; *last* defaults to *first*. + Items that do not support indexing ignore this operation. + + .. method:: insert(tagOrId, beforeThis, string, /) + + Insert *string* into each of the items given by *tagOrId* just before the + character or coordinate whose index is *beforeThis*. + For line and polygon items *string* must be a valid sequence of + coordinates. + + .. method:: itemcget(tagOrId, option) + + Return the current value of the configuration option *option* for the + item given by *tagOrId* (the lowest matching item if it matches several). + This is like :meth:`~Misc.cget` but applies to an individual item. + + .. method:: itemconfig(tagOrId, cnf=None, **kw) + :no-typesetting: + + .. method:: itemconfigure(tagOrId, cnf=None, **kw) + + Query or modify the configuration options of the items given by + *tagOrId*. + This mirrors :meth:`~Misc.configure`, except that it applies to + individual items rather than to the canvas as a whole. + With no options, it returns a dictionary describing the current options + of the first matching item; otherwise it sets the given options on every + matching item. + The legal options are those accepted by the corresponding ``create_*`` + method. + :meth:`itemconfig` is an alias of :meth:`!itemconfigure`. + + .. method:: type(tagOrId) + + Return the type of the item given by *tagOrId* (the first matching item + if it matches several), such as ``'rectangle'`` or ``'text'``, or + ``None`` if *tagOrId* does not match any item. + + .. method:: gettags(tagOrId, /) + + Return a tuple of the tags associated with the item given by *tagOrId* + (the first matching item in display-list order if it matches several). + Return an empty tuple if no item matches or the item has no tags. + + .. method:: dtag(tagOrId, /) + dtag(tagOrId, tagToDelete, /) + + Remove the tag *tagToDelete* (which defaults to *tagOrId*) from each of + the items given by *tagOrId*. + Items that do not have that tag are unaffected. + + .. method:: addtag(newtag, searchSpec, /, *args) + + Add the tag *newtag* to each item selected by the search specification + *searchSpec* (and any further *args*). + *searchSpec* is one of ``'above'``, ``'all'``, ``'below'``, + ``'closest'``, ``'enclosed'``, ``'overlapping'`` or ``'withtag'``; the + ``addtag_*`` methods below are convenient wrappers that supply each of + these forms. + + .. method:: addtag_above(newtag, tagOrId) + + Add the tag *newtag* to the item just above (after) *tagOrId* in the + display list. + + .. method:: addtag_all(newtag) + + Add the tag *newtag* to all items in the canvas. + + .. method:: addtag_below(newtag, tagOrId) + + Add the tag *newtag* to the item just below (before) *tagOrId* in the + display list. + + .. method:: addtag_closest(newtag, x, y, halo=None, start=None) + + Add the tag *newtag* to the item closest to the point (*x*, *y*). + If *halo* is given, any item within that distance of the point is treated + as overlapping it. + If *start* is given (a tag or id), select the topmost closest item that + lies below *start* in the display list, which can be used to step through + all the closest items. + + .. method:: addtag_enclosed(newtag, x1, y1, x2, y2) + + Add the tag *newtag* to every item completely enclosed within the + rectangle (*x1*, *y1*, *x2*, *y2*), where *x1* <= *x2* and *y1* <= *y2*. + + .. method:: addtag_overlapping(newtag, x1, y1, x2, y2) + + Add the tag *newtag* to every item that overlaps or is enclosed within + the rectangle (*x1*, *y1*, *x2*, *y2*), where *x1* <= *x2* and *y1* <= + *y2*. + + .. method:: addtag_withtag(newtag, tagOrId) + + Add the tag *newtag* to every item given by *tagOrId*. + + .. method:: find(searchSpec, /, *args) + + Return a tuple of the ids of all items selected by the search + specification *searchSpec* (and any further *args*), in stacking order + with the lowest item first. + The search specification has any of the forms accepted by :meth:`addtag`. + The ``find_*`` methods below are more convenient wrappers around it. + + .. method:: find_above(tagOrId) + + Return a tuple containing the id of the item just above *tagOrId* in the + display list. + + .. method:: find_all() + + Return a tuple of the ids of all items in the canvas, in stacking order. + + .. method:: find_below(tagOrId) + + Return a tuple containing the id of the item just below *tagOrId* in the + display list. + + .. method:: find_closest(x, y, halo=None, start=None) + + Return a tuple containing the id of the item closest to the point (*x*, + *y*). + *halo* and *start* are interpreted as for :meth:`addtag_closest`. + + .. method:: find_enclosed(x1, y1, x2, y2) + + Return a tuple of the ids of all items completely enclosed within the + rectangle (*x1*, *y1*, *x2*, *y2*). + + .. method:: find_overlapping(x1, y1, x2, y2) + + Return a tuple of the ids of all items that overlap or are enclosed + within the rectangle (*x1*, *y1*, *x2*, *y2*). + + .. method:: find_withtag(tagOrId) + + Return a tuple of the ids of all items given by *tagOrId*. + + .. method:: lift(tagOrId, aboveThis=None, /) + :no-typesetting: + + .. method:: tkraise(tagOrId, aboveThis=None, /) + :no-typesetting: + + .. method:: tag_raise(tagOrId, aboveThis=None, /) + + Move all items given by *tagOrId* to a new position in the display list + just above the item given by *aboveThis*, or to the top of the display + list if *aboveThis* is omitted. + When several items are moved their relative order is preserved. + This has no effect on embedded window items, whose stacking order is + controlled by :meth:`Misc.tkraise` and :meth:`Misc.lower` instead. + :meth:`lift` and :meth:`tkraise` are aliases of :meth:`!tag_raise`. + + .. method:: lower(tagOrId, belowThis=None, /) + :no-typesetting: + + .. method:: tag_lower(tagOrId, belowThis=None, /) + + Move all items given by *tagOrId* to a new position in the display list + just below the item given by *belowThis*, or to the bottom of the display + list if *belowThis* is omitted. + When several items are moved their relative order is preserved. + This has no effect on embedded window items. + :meth:`lower` is an alias of :meth:`!tag_lower`. + + .. method:: tag_bind(tagOrId, sequence=None, func=None, add=None) + + Bind the callback *func* to the event *sequence* for all items given by + *tagOrId*, so that *func* is invoked whenever that event occurs for one + of the items. + This is like :meth:`Widget.bind ` but operates on canvas items + rather than on whole widgets; only mouse, keyboard and virtual events may + be bound. + Mouse events are directed to the current item and keyboard events to the + focus item (see :meth:`focus`). + If *add* is true the new binding is added to any existing bindings for + the same sequence, rather than replacing them. + Return the identifier of the bound function, which can be passed to + :meth:`tag_unbind`. + + .. method:: tag_unbind(tagOrId, sequence, funcid=None) + + Remove for all items given by *tagOrId* the binding for the event + *sequence*. + If *funcid* is given, only that callback (as returned by + :meth:`tag_bind`) is unbound and deregistered. + + .. versionchanged:: 3.13 + If *funcid* is given, only that callback is unbound. + + + .. method:: bbox(tagOrId, /, *tagOrIds) + + Return a 4-tuple ``(x1, y1, x2, y2)`` giving an approximate bounding box, + in pixels, that encloses all the items given by *tagOrId* and any further + *tagOrIds*. + The result may overestimate the true bounding box by a few pixels. + Return ``None`` if no item matches or the matching items have nothing to + display. + + .. method:: canvasx(screenx, gridspacing=None) + + Given a window x-coordinate *screenx*, return the canvas x-coordinate + displayed at that location. + If *gridspacing* is given, the result is rounded to the nearest multiple + of *gridspacing* units. + + .. method:: canvasy(screeny, gridspacing=None) + + Given a window y-coordinate *screeny*, return the canvas y-coordinate + displayed at that location. + If *gridspacing* is given, the result is rounded to the nearest multiple + of *gridspacing* units. + + .. method:: focus() + focus(tagOrId, /) + + With *tagOrId*, set the keyboard focus for the canvas to the first item + given by *tagOrId* that supports the insertion cursor; the focus is left + unchanged if no such item exists. + If *tagOrId* is an empty string, reset the focus so that no item has it. + With no argument, return the id of the item that currently has the focus, + or an empty string if none does. + An item only displays the insertion cursor when both it is the focus item + and its canvas has the input focus. + + .. method:: icursor(tagOrId, index, /) + + Set the insertion cursor of the items given by *tagOrId* to just before + the character given by *index*. + Items that do not support an insertion cursor are unaffected. + The cursor is only displayed when the item has the focus, but its + position may be set at any time. + + .. method:: index(tagOrId, index, /) + + Return as an integer the numerical index within *tagOrId* corresponding + to *index*, which is a textual description of a position (for text items + an index into the characters, for line and polygon items an index into + the coordinates). + If *tagOrId* matches several items, the first one that supports indexing + is used. + + .. method:: select_adjust(tagOrId, index) + + Adjust the end of the selection in *tagOrId* nearest to *index* so that + it is at *index*, and make the other end the anchor point for future + :meth:`select_to` calls. + If the selection is not currently in *tagOrId*, this behaves like + :meth:`select_to`. + + .. method:: select_clear() + + Clear the selection if it is in this canvas; otherwise do nothing. + + .. method:: select_from(tagOrId, index) + + Set the selection anchor point to just before the character given by + *index* in the item given by *tagOrId*. + This does not change the selection itself; it sets the fixed end for + future :meth:`select_to` calls. + + .. method:: select_item() + + Return the id of the item that holds the selection, or ``None`` if the + selection is not in this canvas. + Unlike :meth:`find` and the ``find_*`` methods, this returns the id as a + string rather than an integer. + + .. method:: select_to(tagOrId, index) + + Set the selection to the characters of *tagOrId* between the selection + anchor point and *index*, inclusive of *index*. + The anchor point is the one set by the most recent :meth:`select_adjust` + or :meth:`select_from` call. + + .. method:: scan_mark(x, y) + + Record *x*, *y* and the current view, for use with later + :meth:`scan_dragto` calls. + This is typically bound to a mouse button press in the widget. + + .. method:: scan_dragto(x, y, gain=10) + + Scroll the canvas by *gain* times the difference between *x*, *y* and the + coordinates passed to the last :meth:`scan_mark` call. + This is typically bound to mouse motion events in the widget, producing + the effect of dragging the canvas at high speed through its window. + + .. method:: postscript(cnf={}, **kw) + + Generate a PostScript (Encapsulated PostScript, version 3.0) + representation of part or all of the canvas. + If the *file* or *channel* option is given, the PostScript is written + there and an empty string is returned; otherwise it is returned as a + string. + By default only the area currently visible in the window is generated, so + it is usually necessary either to call :meth:`~Misc.update` first or to + use the *width* and *height* options. + Supported options include *colormap*, *colormode*, *file*, *fontmap*, + *height*, *pageanchor*, *pageheight*, *pagewidth*, *pagex*, *pagey*, + *rotate*, *width*, *x* and *y*. + + +.. class:: Checkbutton(master=None, cnf={}, **kw) + + A :class:`!Checkbutton` widget displays a textual string, bitmap or image + together with a square indicator, and toggles a boolean selection when + pressed. + It has all the behavior of a simple button and, in addition, can be + selected: when selected the indicator is drawn with a check mark and the + associated variable is set to the ``onvalue``, and when deselected the + indicator is drawn empty and the variable is set to the ``offvalue``. + Inherits from :class:`Widget`. + In addition to the standard widget options, a checkbutton accepts the + options documented in the Tk ``checkbutton`` manual page, such as + *variable*, *onvalue*, *offvalue* and *command*. + + .. method:: invoke() + + Do just what would happen if the user pressed the checkbutton with the + mouse: toggle the selection state of the button and invoke the associated + command, if there is one. + Return the result of the command, or an empty string if no command is + associated with the checkbutton. + This is ignored if the checkbutton's state is ``disabled``. + + .. method:: select() + + Select the checkbutton and set the associated variable to its + ``onvalue``. + + .. method:: deselect() + + Deselect the checkbutton and set the associated variable to its + ``offvalue``. + + .. method:: toggle() + + Toggle the selection state of the button, redisplaying it and modifying + its associated variable to reflect the new state. + + .. method:: flash() + + Flash the checkbutton by redisplaying it several times, alternating + between the active and normal colors. + At the end of the flash the checkbutton is left in the same normal or + active state as when the method was called. + This is ignored if the checkbutton's state is ``disabled``. + + +.. class:: Entry(master=None, cnf={}, **kw) + + An :class:`!Entry` widget displays a single line of text and lets the user + edit it. + Inherits from :class:`Widget` and :class:`XView`; since entries can hold + strings too long to fit in the window, they support horizontal scrolling + through :meth:`~XView.xview`. + + In addition to the standard widget options, an entry accepts the options + documented in the Tk ``entry`` manual page. + Notable ones are *textvariable* (the name of a variable kept in sync with + the entry's contents), *show* (if set, each character is displayed as the + given character rather than its true value, useful for password entry), + *validate* and *validatecommand* (which together let a callback accept or + reject edits), and *state* (one of ``'normal'``, ``'disabled'`` or + ``'readonly'``). + + Many of the methods below take an *index* argument that selects a character + in the entry's string. + As described in the Tk ``entry`` manual page, *index* may be a number + (counting from 0), ``'insert'`` (the character just after the insertion + cursor), ``'end'`` (just after the last character), ``'anchor'`` (the + selection anchor point), ``'sel.first'`` and ``'sel.last'`` (the ends of the + selection), or ``@x`` (the character covering pixel x-coordinate *x* in the + window). + Out-of-range indices are rounded to the nearest legal value. + + .. method:: delete(first, last=None) + + Delete the characters from index *first* up to but not including index + *last*. + If *last* is omitted, only the single character at *first* is deleted. + + .. method:: get() + + Return the entry's current string. + + .. method:: insert(index, string) + + Insert *string* just before the character given by *index*. + + .. method:: icursor(index) + + Arrange for the insertion cursor to be displayed just before the + character given by *index*. + + .. method:: index(index) + + Return the numerical index corresponding to *index*. + + .. method:: select_adjust(index) + :no-typesetting: + + .. method:: selection_adjust(index) + + Locate the end of the selection nearest to the character given by + *index*, and adjust that end to be at *index* (including but not going + beyond it); the other end becomes the anchor point for future + :meth:`selection_to` calls. + If there is no selection in the entry, a new one is created between + *index* and the most recent anchor point, inclusive. + :meth:`select_adjust` is an alias of :meth:`!selection_adjust`. + + .. method:: select_clear() + :no-typesetting: + + .. method:: selection_clear() + + Clear the selection if it is currently in this widget. + If the selection is not in this widget the method has no effect. + :meth:`select_clear` is an alias of :meth:`!selection_clear`. + + .. method:: select_from(index) + :no-typesetting: + + .. method:: selection_from(index) + + Set the selection anchor point to just before the character given by + *index*, without changing the selection. + :meth:`select_from` is an alias of :meth:`!selection_from`. + + .. method:: select_present() + :no-typesetting: + + .. method:: selection_present() + + Return ``True`` if there are characters selected in the entry, ``False`` + otherwise. + :meth:`select_present` is an alias of :meth:`!selection_present`. + + .. method:: select_range(start, end) + :no-typesetting: + + .. method:: selection_range(start, end) + + Set the selection to include the characters starting with the one indexed + by *start* and ending with the one just before *end*. + If *end* refers to the same character as *start* or an earlier one, the + selection is cleared. + :meth:`select_range` is an alias of :meth:`!selection_range`. + + .. method:: select_to(index) + :no-typesetting: + + .. method:: selection_to(index) + + Set the selection between the anchor point and *index*: if *index* is + before the anchor point, the selection runs from *index* up to but not + including the anchor; if *index* is after it, from the anchor up to but + not including *index*; if they coincide, nothing happens. + The anchor point is the one set by the most recent :meth:`selection_from` + or :meth:`selection_adjust` call. + If there is no selection in the entry, a new one is created using the + most recent anchor point. + :meth:`select_to` is an alias of :meth:`!selection_to`. + + .. method:: scan_mark(x) + + Record *x* and the current view in the entry window, for use with later + :meth:`scan_dragto` calls. + Typically associated with a mouse button press in the widget. + + .. method:: scan_dragto(x) + + Compute the difference between *x* and the *x* given to the last + :meth:`scan_mark` call, and adjust the view left or right by 10 times + that difference. + Typically associated with mouse motion events, to produce the effect of + dragging the entry at high speed through the window. + + +.. class:: Frame(master=None, cnf={}, **kw) + + A :class:`!Frame` widget is a simple container. + Its primary purpose is to act as a spacer or container for complex window + layouts; its only features are its background and an optional 3-D border to + make the frame appear raised or sunken. + Inherits from :class:`Widget`. + Refer to the Tk ``frame`` manual page for the full list of options. + + +.. class:: Label(master=None, cnf={}, **kw) + + A :class:`!Label` widget displays a non-interactive textual string, bitmap + or image. + The displayed text is set with the *text* option or linked to a variable + through *textvariable*, and an image can be shown using the *image* option. + Text must all be in a single font but may occupy multiple lines, and one + character may be underlined with the *underline* option. + Inherits from :class:`Widget`. + Refer to the Tk ``label`` manual page for the full list of options. + + +.. class:: LabelFrame(master=None, cnf={}, **kw) + + A :class:`!LabelFrame` widget is a container that has the features of a + :class:`Frame` plus the ability to display a label. + The label text is set with the *text* option and positioned with + *labelanchor*, or an arbitrary widget may be used as the label by giving it + as the *labelwidget* option. + Inherits from :class:`Widget`. + Refer to the Tk ``labelframe`` manual page for the full list of options. + + +.. class:: Listbox(master=None, cnf={}, **kw) + + A :class:`!Listbox` widget displays a list of single-line text items, one + per line, of which the user can select one or more. + The way the selection behaves is governed by the *selectmode* option, which + is one of ``browse`` (the default; at most one item, which may be dragged + with the mouse), ``single`` (at most one item), ``multiple`` (any number of + items, toggled individually), or ``extended`` (any number of items, + including discontiguous ranges, selected by clicking and dragging). + Inherits from :class:`Widget`, :class:`XView` and :class:`YView`, so the + view can be scrolled horizontally and vertically with :meth:`~XView.xview` + and :meth:`~YView.yview`. + Refer to the Tk ``listbox`` manual page for the full list of options. + + Many of the methods take an *index* argument identifying a particular item. + As described in the Tk ``listbox`` manual page, *index* may be a numeric + index (counting from 0 at the top), ``'active'`` (the item with the location + cursor, set with :meth:`activate`), ``'anchor'`` (the selection anchor, set + with :meth:`selection_anchor`), ``'end'`` (the last item, or for + :meth:`index` and :meth:`insert` the position just after it), or ``@x,y`` + (the item covering pixel coordinates *x*, *y* in the listbox window). + Arguments named *first* and *last* are indices of the same forms. + + .. method:: insert(index, *elements) + + Insert the given *elements* as new items just before the item given by + *index*. + If *index* is ``'end'``, the new items are appended to the end of the + list. + + .. method:: delete(first, last=None) + + Delete the items in the range from *first* to *last* inclusive. + If *last* is omitted, it defaults to *first*, so that a single item is + deleted. + + .. method:: get(first, last=None) + + If *last* is omitted, return the contents of the item given by *first*, + or an empty string if *first* refers to a non-existent item. + If *last* is given, return a tuple of all the items in the range from + *first* to *last* inclusive. + + .. method:: size() + + Return the total number of items in the listbox. + + .. method:: index(index) + + Return the integer index value corresponding to *index*, or ``None`` if + *index* is out of range. + If *index* is ``'end'``, the result is a count of the number of items in + the listbox (not the index of the last item). + + .. method:: bbox(index) + + Return a tuple ``(x, y, width, height)`` describing the bounding box, in + pixels relative to the widget, of the text of the item given by *index*. + Return ``None`` if no part of that item is visible on the screen, or if + *index* refers to a non-existent item; if the item is only partly + visible, the result still gives the full area of the item, including the + parts that are not visible. + + .. method:: nearest(y) + + Given a y-coordinate within the listbox window, return the index of the + visible item nearest to that y-coordinate. + + .. method:: see(index) + + Adjust the view so that the item given by *index* is visible. + If the item is already visible the method has no effect; if it is near an + edge of the window the listbox scrolls just enough to bring it into view + at that edge, otherwise the listbox scrolls to center the item. + + .. method:: activate(index) + + Set the active item to the one given by *index*. + If *index* is outside the range of items, the closest item is activated + instead. + The active item is drawn as specified by the *activestyle* option when + the widget has the input focus, and its index may be retrieved with the + ``'active'`` index. + + .. method:: curselection() + + Return a tuple containing the numerical indices of all of the items that + are currently selected, or an empty tuple if no items are selected. + + .. method:: select_anchor(index) + :no-typesetting: + + .. method:: selection_anchor(index) + + Set the selection anchor to the item given by *index*. + If *index* refers to a non-existent item, the closest item is used. + The selection anchor is the end of the selection that is fixed while + dragging out a selection with the mouse, and may afterwards be referred + to with the ``'anchor'`` index. + :meth:`select_anchor` is an alias of :meth:`!selection_anchor`. + + .. method:: select_clear(first, last=None) + :no-typesetting: + + .. method:: selection_clear(first, last=None) + + Deselect any of the items in the range from *first* to *last* inclusive + that are selected. + The selection state of items outside this range is not changed. + :meth:`select_clear` is an alias of :meth:`!selection_clear`. + + .. method:: select_includes(index) + :no-typesetting: + + .. method:: selection_includes(index) + + Return ``True`` if the item given by *index* is currently selected, + ``False`` otherwise. + :meth:`select_includes` is an alias of :meth:`!selection_includes`. + + .. method:: select_set(first, last=None) + :no-typesetting: + + .. method:: selection_set(first, last=None) + + Select all of the items in the range from *first* to *last* inclusive, + without affecting the selection state of items outside that range. + :meth:`select_set` is an alias of :meth:`!selection_set`. + + .. method:: itemcget(index, option) + + Return the current value of the configuration option *option* for the + item given by *index*. + + .. method:: itemconfig(index, cnf=None, **kw) + :no-typesetting: + + .. method:: itemconfigure(index, cnf=None, **kw) + + Query or modify the configuration options of the item given by *index*. + This mirrors :meth:`~Misc.configure`, except that it applies to an + individual item rather than to the listbox as a whole. + With no options, it returns a dictionary describing the current options + of the item; otherwise it sets the given options. + The supported item options are *background*, *foreground*, + *selectbackground* and *selectforeground*. + :meth:`itemconfig` is an alias of :meth:`!itemconfigure`. + + .. method:: scan_mark(x, y) + + Record *x*, *y* and the current view, for use with later + :meth:`scan_dragto` calls. + This is typically bound to a mouse button press in the widget. + + .. method:: scan_dragto(x, y) + + Scroll the listbox by 10 times the difference between *x*, *y* and the + coordinates passed to the last :meth:`scan_mark` call. + This is typically bound to mouse motion events in the widget, producing + the effect of dragging the list at high speed through the window. + + +.. class:: Menu(master=None, cnf={}, **kw) + + A :class:`!Menu` widget displays a column of entries, each of which may be a + command, a checkbutton, a radiobutton, a cascade (which posts an associated + submenu) or a separator. + Menus are used as the menubar of a toplevel window, as pulldown menus posted + from a cascade entry or menubutton, and as popup menus. + Inherits from :class:`Widget`. + + Many of the entry methods take an *index* argument that selects which entry + to operate on. + As described in the Tk ``menu`` manual page, *index* may be a numeric index + (counting from 0 at the top), ``'active'`` (the currently active entry), + ``'end'`` or ``'last'`` (the bottommost entry), ``'none'`` (no entry at all, + written ``{}`` in Tcl), ``@y`` (the entry covering pixel y-coordinate *y* in + the menu window), or a pattern matched against the labels of the entries + from the top down. + + .. method:: add(itemType, cnf={}, **kw) + + Add a new entry to the bottom of the menu. + *itemType* is one of ``'command'``, ``'cascade'``, ``'checkbutton'``, + ``'radiobutton'`` or ``'separator'`` and determines the type of the new + entry; the remaining options configure it. + The :meth:`!add_command`, :meth:`!add_cascade`, :meth:`!add_checkbutton`, + :meth:`!add_radiobutton` and :meth:`!add_separator` convenience methods + call this method with the corresponding *itemType*. + + The entry is configured by the following options, although not every + option applies to every entry type (a separator accepts none of them): + + *label* + The text to display in the entry. + + *command* + The function to call when the entry is invoked (command, checkbutton + and radiobutton entries). + + *accelerator* + A string displayed at the right of the entry to advertise an + accelerator keystroke; it does not itself create the binding. + + *underline* + The index of a character in the label to underline for keyboard + traversal. + + *state* + One of ``'normal'``, ``'active'`` or ``'disabled'``. + + *image* + An image to display instead of, or together with, the text label. + + *compound* + Where to show the image relative to the text: ``'none'`` (the + default), ``'text'``, ``'image'``, ``'top'``, ``'bottom'``, ``'left'`` + or ``'right'``. + + *bitmap* + A bitmap to display instead of the text label. + + *font* + The font to use for the text. + + *background*, *foreground* + The entry's background and foreground colors in its normal state + (ignored on macOS). + + *activebackground*, *activeforeground* + The background and foreground colors used when the entry is active + (ignored on macOS). + + *columnbreak* + If true, the entry starts a new column instead of being placed below + the previous entry. + + *hidemargin* + If true, the standard margin around the entry is omitted, which is + useful when a menu is used as a palette. + + *menu* + The submenu posted by a cascade entry; it must be a child of this + menu. + + *variable* + The variable associated with a checkbutton or radiobutton entry. + + *onvalue*, *offvalue* + The values stored in *variable* when a checkbutton entry is selected + or cleared. + + *value* + The value stored in *variable* when a radiobutton entry is selected. + + *indicatoron* + Whether to display the indicator of a checkbutton or radiobutton + entry. + + *selectcolor* + The color of the indicator of a checkbutton or radiobutton entry when + it is selected. + + *selectimage* + The image displayed when a checkbutton or radiobutton entry is + selected and *image* is also given. + + .. method:: add_cascade(cnf={}, **kw) + + Add a new cascade entry to the bottom of the menu. + A cascade entry has an associated submenu, given by its *menu* option, + which must be a child of this menu; posting the entry posts the submenu + next to it. + + .. method:: add_checkbutton(cnf={}, **kw) + + Add a new checkbutton entry to the bottom of the menu. + When invoked, a checkbutton entry toggles between its *onvalue* and + *offvalue*, storing the result in its associated *variable*, and displays + an indicator showing whether it is selected. + + .. method:: add_command(cnf={}, **kw) + + Add a new command entry to the bottom of the menu. + A command entry behaves much like a button: when it is invoked, the + callback given by its *command* option is called. + + .. method:: add_radiobutton(cnf={}, **kw) + + Add a new radiobutton entry to the bottom of the menu. + Radiobutton entries sharing the same *variable* form a group of which + only one may be selected at a time; selecting an entry stores its *value* + in the variable. + + .. method:: add_separator(cnf={}, **kw) + + Add a separator to the bottom of the menu. + A separator is displayed as a horizontal dividing line and cannot be + activated or invoked. + + .. method:: insert(index, itemType, cnf={}, **kw) + + Same as :meth:`add`, except that the new entry is inserted just before + the entry given by *index* instead of being appended to the end of the + menu. + *itemType* is one of ``'command'``, ``'cascade'``, ``'checkbutton'``, + ``'radiobutton'`` or ``'separator'``. + The :meth:`!insert_command`, :meth:`!insert_cascade`, + :meth:`!insert_checkbutton`, :meth:`!insert_radiobutton` and + :meth:`!insert_separator` convenience methods call this method with the + corresponding *itemType*. + + .. method:: insert_cascade(index, cnf={}, **kw) + + Insert a new cascade entry before the entry given by *index* (see + :meth:`add_cascade`). + + .. method:: insert_checkbutton(index, cnf={}, **kw) + + Insert a new checkbutton entry before the entry given by *index* (see + :meth:`add_checkbutton`). + + .. method:: insert_command(index, cnf={}, **kw) + + Insert a new command entry before the entry given by *index* (see + :meth:`add_command`). + + .. method:: insert_radiobutton(index, cnf={}, **kw) + + Insert a new radiobutton entry before the entry given by *index* (see + :meth:`add_radiobutton`). + + .. method:: insert_separator(index, cnf={}, **kw) + + Insert a separator before the entry given by *index* (see + :meth:`add_separator`). + + .. method:: delete(index1, index2=None) + + Delete all of the menu entries between *index1* and *index2* inclusive. + If *index2* is omitted, it defaults to *index1*, so that a single entry + is deleted. + Attempts to delete a tear-off entry are ignored; remove it by changing + the *tearoff* option instead. + + .. method:: entrycget(index, option) + + Return the current value of the configuration option *option* for the + entry given by *index*. + + .. method:: entryconfig(index, cnf=None, **kw) + :no-typesetting: + + .. method:: entryconfigure(index, cnf=None, **kw) + + Query or modify the configuration options of the entry given by *index*. + This mirrors :meth:`~Misc.configure`, except that it applies to an + individual entry rather than to the menu as a whole. + With no options, it returns a dictionary describing the current options + of the entry; otherwise it sets the given options. + The supported options are those accepted by :meth:`add` for the entry's + type. + :meth:`entryconfig` is an alias of :meth:`!entryconfigure`. + + .. method:: index(index) + + Return the numerical index corresponding to *index*, or ``None`` if + *index* selects no entry. + + .. method:: type(index) + + Return the type of the entry given by *index*: one of ``'command'``, + ``'cascade'``, ``'checkbutton'``, ``'radiobutton'``, ``'separator'`` or + ``'tearoff'`` (for the tear-off entry). + + .. method:: activate(index) + + Make the entry given by *index* the active entry, redisplaying it with + its active colors, and deactivate any previously active entry. + If *index* selects no entry, or the selected entry is disabled, the menu + ends up with no active entry. + + .. method:: invoke(index) + + Invoke the action of the entry given by *index*, as if it had been + clicked. + Nothing happens if the entry is disabled. + If the entry has a *command* associated with it, the result of that + command is returned; otherwise the result is an empty string. + + .. method:: post(x, y) + + Display the menu on the screen at the root-window coordinates *x* and + *y*, adjusting them if necessary so that the whole menu is visible. + If the *postcommand* option has been specified, it is evaluated before + the menu is posted. + + .. method:: tk_popup(x, y, entry='') + + Post the menu as a popup at the root-window coordinates *x* and *y*. + If *entry* is given, the menu is positioned so that this entry is + displayed under the pointer. + + .. method:: unpost() + + Unmap the menu so that it is no longer displayed, also unposting any + posted lower-level cascaded submenu. + This has no effect on Windows and macOS, which manage the unposting of + menus themselves. + + .. method:: xposition(index) + + Return the x-coordinate, within the menu window, of the leftmost pixel of + the entry given by *index*. + + .. versionadded:: 3.3 + + .. method:: yposition(index) + + Return the y-coordinate, within the menu window, of the topmost pixel of + the entry given by *index*. + + +.. class:: Menubutton(master=None, cnf={}, **kw) + + A :class:`!Menubutton` widget displays a textual string, bitmap or image and + posts an associated :class:`Menu`, given by its *menu* option, when the user + presses it. + Like a :class:`Label` it can show *text*, a *textvariable*, or an *image*, + and the *direction* option controls where the menu appears relative to the + button. + Inherits from :class:`Widget`. + Refer to the Tk ``menubutton`` manual page for the full list of options. + + +.. class:: Message(master=None, cnf={}, **kw) + + A :class:`!Message` widget displays a non-interactive textual string, given + by the *text* option or linked to a variable through *textvariable*. + Unlike a :class:`Label`, it breaks the string into multiple lines in order + to produce a given aspect ratio, choosing line breaks at word boundaries, + and it can justify the text left, centered or right. + Inherits from :class:`Widget`. + Refer to the Tk ``message`` manual page for the full list of options. + + +.. class:: OptionMenu(master, variable, value, *values, **kwargs) + + A helper subclass of :class:`Menubutton` that displays a pop-up menu of + mutually exclusive choices. + *variable* is a :class:`Variable` kept in sync with the selection, *value* + is the initial choice, and *values* are the remaining menu entries. + The keyword argument *command* may be given a callback that is invoked with + the selected value, and the keyword argument *name* sets the Tk widget name. + + .. method:: destroy() + + Destroy the widget, also cleaning up the associated pop-up menu. + + .. versionchanged:: 3.14 + Added support for the *name* keyword argument. + + + +.. class:: PanedWindow(master=None, cnf={}, **kw) + + A :class:`!PanedWindow` is a geometry-manager widget that arranges any + number of child *panes* in a row (when *orient* is ``'horizontal'``) or a + column (when *orient* is ``'vertical'``). + Each pane holds one widget, and each pair of adjacent panes is separated by + a movable *sash* that the user can drag with the mouse to resize the widgets + on either side of it. + Inherits from :class:`Widget`. + + The *orient* option selects the layout direction, *sashwidth* sets the width + of each sash and *sashrelief* its relief. + When *showhandle* is true a small handle is drawn on each sash that the user + can grab to drag it. + Refer to the Tk ``panedwindow`` manual page for the full list of options. + + .. method:: add(child, **kw) + + Add *child* to the panedwindow as a new pane, placed after any existing + panes. + The keyword arguments specify per-pane management options for *child*; + they may be any of the options accepted by :meth:`paneconfigure`. + + .. method:: forget(child) + :no-typesetting: + + .. method:: remove(child) + + Remove the pane containing *child* from the panedwindow. + All geometry management options for *child* are forgotten. + :meth:`forget` is an alias of :meth:`!remove`. + + .. method:: panes() + + Return a tuple of the widgets managed by the panedwindow, one per pane, + in order. + + .. method:: panecget(child, option) + + Return the current value of the management option *option* for the pane + containing *child*. + *option* may be any value allowed by :meth:`paneconfigure`. + + .. method:: paneconfig(tagOrId, cnf=None, **kw) + :no-typesetting: + + .. method:: paneconfigure(tagOrId, cnf=None, **kw) + + Query or modify the management options of the pane containing the widget + *tagOrId*. + With no options, it returns a dictionary describing all of the available + options for the pane; given a single option name as a string, it returns + a description of that one option; otherwise it sets the given options. + The supported options include *after* and *before* (insert the pane after + or before another managed window), *height* and *width* (the outer + dimensions of the window, including any border), *minsize* (the minimum + size in the paned dimension), *padx* and *pady* (extra space to leave on + each side of the window), *sticky* (position or stretch the window within + an oversized pane, using a string of the characters ``n``, ``s``, ``e`` + and ``w``), *hide* (hide the pane while keeping it in the list of panes) + and *stretch* (how extra space is allocated to the pane: one of + ``'always'``, ``'first'``, ``'last'``, ``'middle'`` or ``'never'``). + :meth:`paneconfig` is an alias of :meth:`!paneconfigure`. + + .. method:: identify(x, y) + + Identify the panedwindow component underneath the point given by *x* and + *y*, in window coordinates. + If the point is over a sash or a sash handle, the result is a two-element + tuple containing the index of the sash or handle and a word indicating + whether it is over a sash or a handle, such as ``(0, 'sash')`` or + ``(2, 'handle')``. + If the point is over any other part of the panedwindow, the result is an + empty string. + + .. method:: sash(*args) + + Query or change the position of the sashes in the panedwindow. + This is a thin wrapper around the Tk ``sash`` subcommand; the convenience + methods :meth:`sash_coord`, :meth:`sash_mark` and :meth:`sash_place` + should normally be used instead. + + .. method:: sash_coord(index) + + Return the current x and y coordinate pair for the sash given by *index*, + which must be an integer between 0 and one less than the number of panes + in the panedwindow. + The coordinates returned are those of the top left corner of the region + containing the sash. + + .. method:: sash_mark(index) + + Record the current mouse position for the sash given by *index*, for use + together with later sash-drag operations to move the sash. + + .. method:: sash_place(index, x, y) + + Place the sash given by *index* at the coordinates *x* and *y*. + + .. method:: proxy(*args) + + Query or change the position of the sash proxy, the "ghost" sash shown + while a sash is being dragged with non-opaque resizing. + This is a thin wrapper around the Tk ``proxy`` subcommand; the + convenience methods :meth:`proxy_coord`, :meth:`proxy_forget` and + :meth:`proxy_place` should normally be used instead. + + .. method:: proxy_coord() + + Return a tuple containing the x and y coordinates of the most recent + proxy location. + + .. method:: proxy_forget() + + Remove the proxy from the display. + + .. method:: proxy_place(x, y) + + Place the proxy at the coordinates *x* and *y*. + + +.. class:: Radiobutton(master=None, cnf={}, **kw) + + A :class:`!Radiobutton` widget displays a textual string, bitmap or image + together with a diamond or circular indicator, and selects one choice out of + several. + It has all the behavior of a simple button and, in addition, can be + selected: typically several radiobuttons share a single *variable*, and + selecting one sets that variable to the radiobutton's *value*; each + radiobutton also monitors the variable and automatically selects or + deselects itself when the variable changes. + Inherits from :class:`Widget`. + In addition to the standard widget options, a radiobutton accepts the + options documented in the Tk ``radiobutton`` manual page, such as + *variable*, *value* and *command*. + + .. method:: invoke() + + Do just what would happen if the user pressed the radiobutton with the + mouse: select the button and invoke the associated command, if there is + one. + Return the result of the command, or an empty string if no command is + associated with the radiobutton. + This is ignored if the radiobutton's state is ``disabled``. + + .. method:: select() + + Select the radiobutton and set the associated variable to the value + corresponding to this widget. + + .. method:: deselect() + + Deselect the radiobutton and set the associated variable to an empty + string. + If this radiobutton was not currently selected, this has no effect. + + .. method:: flash() + + Flash the radiobutton by redisplaying it several times, alternating + between the active and normal colors. + At the end of the flash the radiobutton is left in the same normal or + active state as when the method was called. + This is ignored if the radiobutton's state is ``disabled``. + + +.. class:: Scale(master=None, cnf={}, **kw) + + A :class:`!Scale` widget lets the user select a numerical value by moving a + slider along a trough. + It can be oriented vertically or horizontally and can optionally display a + label and the current value. + Inherits from :class:`Widget`. + + In addition to the standard widget options, a scale accepts the options + documented in the Tk ``scale`` manual page, such as *from_*, *to*, + *resolution*, *orient*, *tickinterval*, *variable* and *command*. + As elsewhere in :mod:`!tkinter`, the leading ``-`` of the Tk option name is + dropped; *from* is spelled ``from_`` because :keyword:`from` is a Python + keyword. + + .. method:: get() + + Return the current value of the scale. + The result is an integer if the scale's *resolution* yields whole + numbers, and a float otherwise. + + .. method:: set(value) + + Set the scale to *value*, moving the slider accordingly. + This has no effect if the scale is disabled. + + .. method:: coords(value=None) + + Return a tuple ``(x, y)`` giving the pixel coordinates, relative to the + widget, of the point on the centerline of the trough that corresponds to + *value*. + If *value* is omitted, the scale's current value is used. + + .. method:: identify(x, y) + + Return a string describing the part of the scale at the pixel coordinates + *x*, *y*: ``'slider'``, ``'trough1'`` (the part of the trough above or to + the left of the slider), ``'trough2'`` (below or to the right of the + slider), or an empty string if the point is not over any of these + elements. + + +.. class:: Scrollbar(master=None, cnf={}, **kw) + + A :class:`!Scrollbar` widget displays a slider and two arrows that let the + user scroll an associated widget, such as a :class:`Listbox`, :class:`Text`, + :class:`Canvas` or :class:`Entry`. + It is connected to the scrolled widget by setting that widget's + *xscrollcommand* or *yscrollcommand* option to the scrollbar's :meth:`set` + method, and the scrollbar's *command* option to the scrolled widget's + :meth:`~XView.xview` or :meth:`~YView.yview` method. + Inherits from :class:`Widget`. + + .. method:: get() + + Return the current scrollbar settings as a tuple ``(first, last)`` of two + fractions between 0 and 1, describing the portion of the document that is + currently visible, as last passed to :meth:`set`. + + .. method:: set(first, last) + + Set the scrollbar. + *first* and *last* are fractions between 0 and 1 giving the positions of + the start and end of the visible portion of the associated document. + This method is normally registered as the scrolled widget's + *xscrollcommand* or *yscrollcommand* and called by that widget. + + .. method:: activate(index=None) + + Mark the element *index* (one of ``'arrow1'``, ``'slider'`` or + ``'arrow2'``) as active, displaying it according to the + *activebackground* and *activerelief* options. + If *index* is omitted, return the name of the currently active element, + or ``None`` if no element is active. + + .. versionchanged:: 3.5 + The *index* argument is now optional. + + .. method:: delta(deltax, deltay) + + Return a float indicating the fractional change in the scrollbar setting + that corresponds to moving the slider by *deltax* pixels horizontally + (for horizontal scrollbars) or *deltay* pixels vertically (for vertical + scrollbars). + + .. method:: fraction(x, y) + + Return a float between 0 and 1 indicating where the point at pixel + coordinates *x*, *y* lies in the trough: 0 corresponds to the top or left + of the trough and 1 to the bottom or right. + + .. method:: identify(x, y) + + Return the name of the element under the pixel coordinates *x*, *y* (such + as ``'arrow1'``), or an empty string if the point does not lie in any + element of the scrollbar. + + +.. class:: Spinbox(master=None, cnf={}, **kw) + + A :class:`!Spinbox` widget is an :class:`Entry`-like widget with a pair of + up/down arrow buttons that let the user step through a range of values in + addition to editing the value directly. + The set of values may be a numeric range given by the *from_*, *to* and + *increment* options, or an explicit list of strings given by the *values* + option (which takes precedence over the range). + Each time an arrow is invoked the *command* callback, if any, is called; the + *wrap* option controls whether stepping past either end of the range wraps + around to the other end; the *format* option specifies how numeric values + are formatted; and the *validate* option enables validation of the entered + text. + Inherits from :class:`Widget` and :class:`XView`. + + Many of the methods take an *index* argument identifying a character in the + spinbox's string. + As described in the Tk ``spinbox`` manual page, *index* may be a numeric + index (counting from 0), ``'anchor'`` (the selection anchor point), + ``'end'`` (just after the last character), ``'insert'`` (the character just + after the insertion cursor), ``'sel.first'`` or ``'sel.last'`` (the ends of + the selection), or ``@x`` (the character covering pixel x-coordinate *x* in + the window). + + .. method:: get() + + Return the spinbox's string. + + .. method:: insert(index, s) + + Insert the characters of the string *s* just before the character given + by *index*. + + .. method:: delete(first, last=None) + + Delete one or more characters of the spinbox. + *first* is the index of the first character to delete, and *last* is the + index of the character just after the last one to delete. + If *last* is omitted, a single character at *first* is deleted. + + .. method:: icursor(index) + + Arrange for the insertion cursor to be displayed just before the + character given by *index*. + + .. method:: index(index) + + Return the numerical index corresponding to *index*, as a string. + + .. method:: bbox(index) + + Return a tuple of four integers ``(x, y, width, height)`` describing the + bounding box of the character given by *index*. + *x* and *y* are the pixel coordinates of the upper-left corner of the + character relative to the widget, and *width* and *height* are its size + in pixels. + The bounding box may refer to a region outside the visible area of the + window. + + .. method:: identify(x, y) + + Return the name of the window element at the pixel coordinates *x*, *y*: + one of ``'buttondown'``, ``'buttonup'``, ``'entry'`` or ``'none'``. + + .. method:: invoke(element) + + Invoke the spin button given by *element*, either ``'buttonup'`` or + ``'buttondown'``, triggering the action associated with it. + + .. method:: scan(*args) + + A thin wrapper around the Tk ``scan`` widget subcommand, used to + implement fast dragging of the view: ``scan('mark', x)`` records *x* and + the current view, and ``scan('dragto', x)`` adjusts the view relative to + that mark. + The :meth:`scan_mark` and :meth:`scan_dragto` methods wrap the two forms. + + .. method:: scan_mark(x) + + Record *x* and the current view in the spinbox window, for use with a + later :meth:`scan_dragto` call. + This is typically associated with a mouse button press in the widget. + + .. method:: scan_dragto(x) + + Adjust the view by 10 times the difference between *x* and the *x* passed + to the last :meth:`scan_mark` call. + This is typically associated with mouse motion events, producing the + effect of dragging the spinbox at high speed through the window. + + .. method:: selection(*args) + + A thin wrapper around the Tk ``selection`` widget subcommand, used to + adjust the selection within the spinbox. + It has several forms depending on the first argument, such as + ``selection('adjust', index)``, ``selection('clear')``, + ``selection('element', ?elem?)``, ``selection('from', index)``, + ``selection('present')``, ``selection('range', start, end)`` and + ``selection('to', index)``. + The :meth:`selection_adjust`, :meth:`selection_clear`, + :meth:`selection_element`, :meth:`selection_from`, + :meth:`selection_present`, :meth:`selection_range` and + :meth:`selection_to` methods wrap these forms. + + .. method:: selection_adjust(index) + + Locate the end of the selection nearest to the character given by *index* + and adjust that end of the selection to be at *index* (including but not + going beyond *index*). + The other end becomes the anchor point for future :meth:`selection_to` + calls. + If the selection is not currently in the spinbox, a new selection is + created to include the characters between *index* and the most recent + anchor point, inclusive. + + .. method:: selection_clear() + + Clear the selection if it is currently in this widget. + If the selection is not in this widget, the method has no effect. + + .. method:: selection_element(element=None) + + Set or get the currently selected element. + If *element* (one of ``'buttonup'``, ``'buttondown'`` or ``'none'``) is + given, that spin button is selected and displayed depressed; otherwise + the name of the currently selected element is returned. + + .. method:: selection_from(index) + + Set the selection anchor point to just before the character given by + *index*, without changing the selection itself. + + .. versionadded:: 3.8 + + + .. method:: selection_present() + + Return ``True`` if there are characters selected in the spinbox, + ``False`` otherwise. + + .. versionadded:: 3.8 + + + .. method:: selection_range(start, end) + + Set the selection to include the characters starting with the one indexed + by *start* and ending with the one just before *end*. + If *end* refers to the same character as *start* or an earlier one, the + selection is cleared. + + .. versionadded:: 3.8 + + + .. method:: selection_to(index) + + Set the selection between *index* and the anchor point. + If *index* is before the anchor point, the selection runs from *index* up + to but not including the anchor point; if it is after, the selection runs + from the anchor point up to but not including *index*; if it is the same, + nothing happens. + The anchor point is the one set by the most recent :meth:`selection_from` + or :meth:`selection_adjust` call. + If the selection is not in this widget, a new selection is created using + the most recent anchor point. + + .. versionadded:: 3.8 + + + +.. class:: Text(master=None, cnf={}, **kw) + + A :class:`!Text` widget displays and edits multi-line text. + Portions of the text may be styled with **tags**, particular positions may + be annotated with floating **marks**, and arbitrary images and other widgets + may be embedded in the text. + The widget also provides an unlimited undo/redo mechanism and supports peer + widgets that share the same underlying data. + Inherits from :class:`Widget`, :class:`XView` and :class:`YView`, so the + view can be scrolled horizontally and vertically with :meth:`~XView.xview` + and :meth:`~YView.yview`. + Refer to the Tk ``text`` manual page for the full list of options. + + Most of the methods take one or more *index* arguments that identify a + position within the text. + As described in the Tk ``text`` manual page, an index is a string consisting + of a base, optionally followed by one or more modifiers. + The base may be ``'line.char'`` (line *line*, character *char*, where lines + are counted from 1 and characters within a line from 0; ``'line.end'`` + refers to the newline ending the line), ``'end'`` (the position just after + the last newline), the name of a mark, ``'tag.first'`` or ``'tag.last'`` + (the first character tagged with *tag*, or the position just after the last + such character), the name of an embedded image or window, or ``@x,y`` (the + character covering pixel coordinates *x*, *y* in the widget). + A modifier such as ``'+5 chars'``, ``'-3 lines'``, ``'linestart'``, + ``'lineend'``, ``'wordstart'`` or ``'wordend'`` adjusts the index relative + to its base; several modifiers may be combined and are applied from left to + right, for example ``'insert wordstart - 1 c'``. + + .. method:: insert(index, chars, *args) + + Insert the string *chars* just before the character at *index* (if + *index* is ``'end'``, just before the final newline). + By default the new text inherits any tags present on both sides of the + insertion point. + If *args* is given, it consists of alternating *tagList*, *chars* values: + the preceding *chars* receives exactly the tags listed (a tag list may be + a single tag name or a sequence of names), overriding the surrounding + tags. + + .. method:: delete(index1, index2=None) + + Delete the range of characters from *index1* up to but not including + *index2*. + If *index2* is omitted, the single character at *index1* is deleted. + The widget always keeps a newline as its last character, so a deletion + that would remove it is adjusted accordingly. + + .. method:: replace(index1, index2, chars, *args) + + Replace the range of characters from *index1* up to but not including + *index2* with *chars*. + This is equivalent to a :meth:`delete` followed by an :meth:`insert` at + *index1*; *args* is interpreted as for :meth:`insert`. + + .. versionadded:: 3.3 + + .. method:: get(index1, index2=None) + + Return the text from *index1* up to but not including *index2* as a + string. + If *index2* is omitted, return the single character at *index1*. + Embedded images and windows are omitted from the result. + + .. method:: index(index) + + Return the position corresponding to *index* in the canonical + ``'line.char'`` form. + + .. method:: compare(index1, op, index2) + + Compare the positions of *index1* and *index2* using the relational + operator *op*, which must be one of ``'<'``, ``'<='``, ``'=='``, + ``'>='``, ``'>'`` or ``'!='``, and return the boolean result. + + .. method:: count(index1, index2, *options, return_ints=False) + + Count the number of items of the requested kinds between *index1* and + *index2*; the count is negative if *index1* is after *index2*. + Each of *options* names a kind of item to count: ``'chars'``, + ``'displaychars'``, ``'displayindices'``, ``'displaylines'``, + ``'indices'``, ``'lines'``, ``'xpixels'`` or ``'ypixels'`` (the default, + used when no option is given, is ``'indices'``). + The pseudo-option ``'update'`` forces any out-of-date layout information + to be recalculated before the following options are evaluated. + When *return_ints* is true and a single counting option is given, return + a plain integer; otherwise return a tuple with one integer per counting + option (or ``None`` if the result is empty). + + .. versionadded:: 3.3 + + .. versionchanged:: 3.13 + Added the *return_ints* parameter. + + + .. method:: see(index) + + Adjust the view so that the character given by *index* is visible. + If it is already visible the method has no effect; if it is a short + distance out of view the widget scrolls just enough to bring it to the + nearest edge, otherwise it scrolls to center *index* in the window. + + .. method:: bbox(index) + + Return a tuple ``(x, y, width, height)`` giving the bounding box, in + pixels, of the visible part of the character at *index*, or ``None`` if + that character is not visible on the screen. + + .. method:: dlineinfo(index) + + Return a tuple ``(x, y, width, height, baseline)`` describing the display + line that contains *index*: the first four values give the bounding box + of the line in pixels and *baseline* gives the offset of the baseline + measured down from the top of the area. + Return ``None`` if that display line is not visible on the screen. + + .. method:: mark_set(markName, index) + + Set the mark named *markName* to the position just before the character + at *index*, creating the mark if it does not already exist. + A mark created this way has right gravity by default. + + .. method:: mark_unset(*markNames) + + Remove each of the marks named in *markNames*. + The special ``insert`` and ``current`` marks may not be removed. + + .. method:: mark_names() + + Return a tuple of the names of all marks currently set in the widget. + + .. method:: mark_gravity(markName, direction=None) + + If *direction* is omitted, return the gravity of mark *markName*, either + ``'left'`` or ``'right'``. + Otherwise set its gravity to *direction*. + The gravity determines on which side of the mark text inserted at the + mark's position appears: a mark with right gravity (the default) stays to + the right of such text. + + .. method:: mark_next(index) + + Return the name of the first mark at or after *index*, or ``None`` if + there is none. + When *index* is the name of a mark, the search starts just after that + mark. + + .. method:: mark_previous(index) + + Return the name of the last mark at or before *index*, or ``None`` if + there is none. + When *index* is the name of a mark, the search starts just before that + mark. + + .. method:: tag_add(tagName, index1, *args) + + Add the tag *tagName* to the range of characters from *index1* up to but + not including the next index in *args*. + Further pairs of indices may follow in *args* to tag additional ranges; a + trailing single index tags just the character at that index. + + .. method:: tag_remove(tagName, index1, index2=None) + + Remove the tag *tagName* from the characters from *index1* up to but not + including *index2* (or from the single character at *index1* if *index2* + is omitted). + The tag itself continues to exist even if no characters carry it. + + .. method:: tag_delete(*tagNames) + + Delete each of the tags named in *tagNames*, removing them from all + characters and discarding their options and bindings. + + .. method:: tag_config(tagName, cnf=None, **kw) + :no-typesetting: + + .. method:: tag_configure(tagName, cnf=None, **kw) + + Query or modify the configuration options of the tag *tagName*. + This mirrors :meth:`~Misc.configure`, except that it applies to a tag + rather than to the widget as a whole: with no options it returns a + dictionary describing the current options, otherwise it sets the given + options. + Defining a tag this way also gives it a priority higher than any existing + tag. + + The supported tag options, all controlling the appearance of the tagged + text, are: + + *font* + The font to use for the text. + + *foreground* + The color to use for the text. + + *background* + The color to use for the area behind the text. + + *fgstipple*, *bgstipple* + Bitmaps used to stipple the foreground (text) and the background; + only well supported on X11. + + *borderwidth* + The width of the border drawn around the text according to *relief* + (default ``0``). + + *relief* + The 3-D appearance of the text's border: ``'flat'`` (the default), + ``'raised'``, ``'sunken'``, ``'ridge'``, ``'groove'`` or ``'solid'``. + + *offset* + How far the text is raised above (or, if negative, lowered below) the + baseline, for superscripts and subscripts. + + *underline* + Whether to underline the text. + + *underlinefg* + The color of the underline; it defaults to the text color. + + *overstrike* + Whether to draw a line through the middle of the text. + + *overstrikefg* + The color of the overstrike line; it defaults to the text color. + + *elide* + Whether the text is elided (hidden). + + *justify* + How to justify the first character of a display line: ``'left'`` (the + default), ``'right'`` or ``'center'``. + + *wrap* + How to wrap lines that are too long: ``'char'``, ``'word'`` or + ``'none'``. + + *lmargin1*, *lmargin2* + The indentation, in pixels, of the first display line of a logical + line and of the remaining display lines. + + *lmargincolor* + The color of the left margin area. + + *rmargin* + The right-hand margin, in pixels. + + *rmargincolor* + The color of the right margin area. + + *spacing1*, *spacing2*, *spacing3* + Extra space, in pixels, above the first display line of a logical + line, between its display lines, and below its last display line. + + *tabs* + The set of tab stops, in the same form as the widget's *tabs* option. + + *tabstyle* + How tab stops are interpreted: ``'tabular'`` or ``'wordprocessor'``. + + *selectbackground*, *selectforeground* + The background and foreground colors used for the text while it is + selected. + + .. note:: + + Tk 8.6 added the *lmargincolor*, *overstrikefg*, *rmargincolor*, + *selectbackground*, *selectforeground* and *underlinefg* options. + + :meth:`tag_config` is an alias of :meth:`!tag_configure`. + + .. method:: tag_cget(tagName, option) + + Return the current value of the configuration option *option* for the tag + *tagName*. + + .. method:: tag_names(index=None) + + If *index* is omitted, return a tuple of the names of all tags defined in + the widget; otherwise return only the names of the tags applied to the + character at *index*. + The names are ordered from lowest to highest priority. + + .. method:: tag_ranges(tagName) + + Return a tuple of indices describing all ranges of text tagged with + *tagName*. + The result alternates start and end indices, so that elements ``2*i`` and + ``2*i+1`` bound the *i*-th range. + + .. method:: tag_nextrange(tagName, index1, index2=None) + + Search forward from *index1* (up to *index2* if given) for the first + range of characters tagged with *tagName*, and return a two-element tuple + of its start and end indices, or an empty tuple if there is no such + range. + + .. method:: tag_prevrange(tagName, index1, index2=None) + + Search backward from *index1* (down to *index2* if given) for the nearest + preceding range of characters tagged with *tagName*, and return a + two-element tuple of its start and end indices, or an empty tuple if + there is no such range. + + .. method:: tag_raise(tagName, aboveThis=None) + + Raise the priority of tag *tagName* so that it is just above the priority + of *aboveThis*, or to the highest priority of all tags if *aboveThis* is + omitted. + When the display options of overlapping tags conflict, the + higher-priority tag wins. + + .. method:: tag_lower(tagName, belowThis=None) + + Lower the priority of tag *tagName* so that it is just below the priority + of *belowThis*, or to the lowest priority of all tags if *belowThis* is + omitted. + + .. method:: tag_bind(tagName, sequence, func, add=None) + + Bind the event *sequence* on characters tagged with *tagName* to the + callback *func*, so that *func* is invoked when that event occurs over + such a character. + If *add* is true the binding is added alongside any existing bindings for + *sequence*, otherwise it replaces them. + Works like :meth:`~Misc.bind` and returns the identifier of the new + binding. + + .. method:: tag_unbind(tagName, sequence, funcid=None) + + Remove the bindings of the event *sequence* on characters tagged with + *tagName*. + If *funcid* is given, only that binding (as returned by :meth:`tag_bind`) + is removed and its callback is unregistered. + + .. versionchanged:: 3.13 + If *funcid* is given, only that callback is unbound. + + + .. method:: image_create(index, cnf={}, **kw) + + Embed an image at *index* and return the name assigned to this image + instance, which may then be used as an index or passed to the other + ``image_*`` methods. + The options, given in *cnf* and *kw*, include *image* (the Tk image to + display), *name* (a base name for the instance), *align*, *padx* and + *pady*. + + .. method:: image_cget(index, option) + + Return the current value of the configuration option *option* for the + embedded image at *index*. + + .. method:: image_configure(index, cnf=None, **kw) + + Query or modify the configuration options of the embedded image at + *index*, like :meth:`~Misc.configure` but applied to that image. + + .. method:: image_names() + + Return a tuple of the names of all images embedded in the widget. + + .. method:: window_create(index, cnf={}, **kw) + + Embed a window (any widget) at *index*. + The options, given in *cnf* and *kw*, include *window* (the widget to + embed), *create* (a callback that creates the widget on demand), *align*, + *stretch*, *padx* and *pady*. + The embedded widget must be a descendant of the text widget's parent. + + .. method:: window_cget(index, option) + + Return the current value of the configuration option *option* for the + embedded window at *index*. + + .. method:: window_config(index, cnf=None, **kw) + :no-typesetting: + + .. method:: window_configure(index, cnf=None, **kw) + + Query or modify the configuration options of the embedded window at + *index*, like :meth:`~Misc.configure` but applied to that window. + + :meth:`window_config` is an alias of :meth:`!window_configure`. + + .. method:: window_names() + + Return a tuple of the names of all windows embedded in the widget. + + .. method:: edit(*args) + + Low-level wrapper around the Tk ``edit`` widget command that controls the + undo/redo mechanism and the modified flag; *args* is the ``edit`` + subcommand and its arguments. + The :meth:`!edit_\*` methods below are thin wrappers around it and are + usually more convenient. + + .. method:: edit_modified(arg=None) + + If *arg* is omitted, return the current state of the modified flag as + ``0`` or ``1``; the flag is set automatically whenever the text is + inserted or deleted. + Otherwise set the flag to the boolean *arg*. + + .. method:: edit_canundo() + + Return ``True`` if there is an edit action on the undo stack that can be + undone, and ``False`` otherwise. + + .. versionadded:: next + + .. method:: edit_canredo() + + Return ``True`` if there is an edit action on the redo stack that can be + reapplied, and ``False`` otherwise. + + .. versionadded:: next + + .. method:: edit_undo() + + Undo the most recent edit action, that is, all the inserts and deletes + recorded on the undo stack since the previous separator, and move it to + the redo stack. + Raises :exc:`TclError` if the undo stack is empty. + Has no effect unless the *undo* option is true. + Since Tk 9.0, returns a tuple of indices delimiting the ranges of text + that were changed. + + .. method:: edit_redo() + + Reapply the most recently undone edit action, provided no further edits + have been made since, and move it back to the undo stack. + Raises :exc:`TclError` if the redo stack is empty. + Has no effect unless the *undo* option is true. + Since Tk 9.0, returns a tuple of indices delimiting the ranges of text + that were changed. + + .. method:: edit_reset() + + Clear the undo and redo stacks. + + .. method:: edit_separator() + + Push a separator onto the undo stack, marking a boundary between edit + actions for undo and redo. + Has no effect unless the *undo* option is true. + Separators are inserted automatically when the *autoseparators* option is + true. + + .. method:: search(pattern, index, stopindex=None, forwards=None, backwards=None, exact=None, regexp=None, nocase=None, count=None, elide=None, *, nolinestop=None, strictlimits=None) + + Search for *pattern* starting at *index* and return the index of the + first character of the first match, or an empty string if there is no + match. + Searching stops at *stopindex* if given; otherwise it wraps around the + ends of the text until the starting position is reached again. + The following boolean keyword flags control the search: *forwards* or + *backwards* select the direction (forward is the default); *exact* (the + default) or *regexp* select literal or regular-expression matching; + *nocase* makes the match case-insensitive; *elide* causes hidden text to + be searched as well; *nolinestop* (regexp only) lets ``.`` and ``[^`` + match newlines; and *strictlimits* requires the whole match to lie within + *index* and *stopindex*. + If *count* is a :class:`Variable`, the number of index positions in the + match is stored in it. + + .. versionchanged:: 3.15 + Added the *nolinestop* and *strictlimits* parameters. + + + .. method:: search_all(pattern, index, stopindex=None, *, forwards=None, backwards=None, exact=None, regexp=None, nocase=None, count=None, elide=None, nolinestop=None, overlap=None, strictlimits=None) + + Like :meth:`search`, but find every match in the searched range and + return a tuple of the starting indices of all matches (empty if there are + none). + By default overlapping matches are not reported; passing a true *overlap* + returns every match that is not wholly contained in another. + If *count* is a :class:`Variable`, it receives a list with one element + per match. + + .. versionadded:: 3.15 + + + .. method:: scan_mark(x, y) + + Record *x*, *y* and the current view, for use with later + :meth:`scan_dragto` calls. + This is typically bound to a mouse button press in the widget. + + .. method:: scan_dragto(x, y) + + Scroll the widget by 10 times the difference between *x*, *y* and the + coordinates passed to the last :meth:`scan_mark` call. + This is typically bound to mouse motion events, producing the effect of + dragging the text at high speed through the window. + + .. method:: debug(boolean=None) + + If *boolean* is omitted, return whether internal consistency checks of + the B-tree data structure are enabled. + Otherwise enable or disable them. + The setting is shared by all text widgets and may noticeably slow down + widgets holding large amounts of text. + + .. method:: dump(index1, index2=None, command=None, **kw) + + Return the contents of the widget from *index1* up to but not including + *index2* (or just the segment at *index1* if *index2* is omitted), + including text and information about marks, tags, images and windows. + The result is a list of ``(key, value, index)`` triples, where *key* is + one of ``'text'``, ``'mark'``, ``'tagon'``, ``'tagoff'``, ``'image'`` or + ``'window'``. + By default all kinds are reported; passing any of the keyword arguments + *all*, *text*, *mark*, *tag*, *image* or *window* as true restricts the + dump to the selected kinds. + If *command* is given, it is called once per triple with the three values + as arguments and nothing is returned. + + .. method:: peer_create(newPathName, cnf={}, **kw) + + Create a peer text widget with the path name *newPathName* that shares + this widget's underlying data (text, marks, tags, images and the undo + stack). + Changes made through any peer are reflected in all of them. + By default the peer covers the same lines as this widget; standard text + options, including *startline* and *endline*, may be given to override + this. + + .. versionadded:: 3.3 + + .. method:: peer_names() + + Return a tuple of the path names of this widget's peers, not including + the widget itself. + + .. versionadded:: 3.3 + + .. method:: sync(command=None) + + Control the synchronization of the displayed view with the underlying + text, which may lag behind when line heights have not yet been computed + (for example, for lines that have never been displayed). + If *command* is omitted, bring the line metrics up to date immediately by + forcing computation of any outdated line heights, and return once they + are current. + Otherwise schedule *command* to be called, with no arguments, exactly + once as soon as all line heights are up to date; if there are no pending + calculations, it is called immediately. + + .. versionadded:: next + + .. method:: pendingsync() + + Return ``True`` if the line height calculations are not up to date, and + ``False`` otherwise. + The ``<>`` virtual event fires whenever this state + changes, with the *detail* field set to the new value. + + .. versionadded:: next + + .. method:: yview_pickplace(*what) + + Adjust the view so that the location given by *what* is visible. + This is an obsolete equivalent of :meth:`see`, which should be used + instead. + + +Variable classes +^^^^^^^^^^^^^^^^ + +.. class:: Variable(master=None, value=None, name=None) + + The base class for the Tk variable wrappers. + A Tk variable is a value stored in the Tcl interpreter that can be linked to + widgets through their *variable* or *textvariable* options (see + :ref:`coupling-widget-variables`), so that changes propagate both ways: + updating the variable updates every widget bound to it, and a user editing + such a widget updates the variable. + + *master* is the widget whose Tcl interpreter owns the variable; if omitted, + the default root window is used. + *value* is the initial value; if omitted, a type-specific default is used. + *name* is the name of the variable in the Tcl interpreter; if omitted, a + unique name of the form ``'PY_VARnum'`` is generated. + If *name* matches an existing variable and *value* is omitted, the existing + value is retained. + + In most cases you should use one of the typed subclasses below -- + :class:`StringVar`, :class:`IntVar`, :class:`DoubleVar` or + :class:`BooleanVar` -- rather than :class:`!Variable` directly. + + .. versionchanged:: 3.10 + Two variables now compare equal (``==``) only when they have the same + name, are of the same class, and belong to the same Tcl interpreter. + + .. method:: get() + + Return the current value of the variable. + For the base class the value is returned as a string; the typed + subclasses convert it to the appropriate Python type. + + .. method:: initialize(value) + :no-typesetting: + + .. method:: set(value) + + Set the variable to *value*. + + .. versionadded:: 3.3 + The *initialize* spelling. + + .. method:: trace_add(mode, callback) + + Register *callback* to be called when the variable is accessed according + to *mode*. + *mode* is one of the strings ``'array'``, ``'read'``, ``'write'`` or + ``'unset'``, or a list or tuple of such strings. + + When triggered, *callback* is called with three arguments: the name of + the Tcl variable, an index (or an empty string if the variable is not an + element of an array), and the *mode* that triggered the call. + + Return the internal name of the registered callback, which can be passed + to :meth:`trace_remove`. + + .. versionadded:: 3.6 + + .. method:: trace_remove(mode, cbname) + + Remove a trace callback from the variable. + *mode* must match the *mode* that was passed to :meth:`trace_add`, and + *cbname* is the callback name returned by :meth:`trace_add`. + + .. versionadded:: 3.6 + + .. method:: trace_info() + + Return a list of ``(modes, cbname)`` pairs describing all traces + currently set on the variable, where *modes* is a tuple of mode strings + and *cbname* is the internal callback name. + + .. versionadded:: 3.6 + + .. method:: trace(mode, callback) + :no-typesetting: + + .. method:: trace_variable(mode, callback) + + Register *callback* to be called when the variable is accessed according + to *mode*. + *mode* is one of the strings ``'r'``, ``'w'`` or ``'u'``, for read, write + or unset. + Return the internal name of the registered callback. + + .. deprecated:: 3.6 + Use :meth:`trace_add` instead. This method wraps a Tcl feature that + was removed in Tcl 9.0. + + .. method:: trace_vdelete(mode, cbname) + + Remove the trace callback named *cbname* registered for *mode* with + :meth:`trace_variable`. + + .. deprecated:: 3.6 + Use :meth:`trace_remove` instead. This method wraps a Tcl feature + that was removed in Tcl 9.0. + + .. method:: trace_vinfo() + + Return a list of ``(mode, cbname)`` pairs for all traces set on the + variable with :meth:`trace_variable`. + + .. deprecated:: 3.6 + Use :meth:`trace_info` instead. This method wraps a Tcl feature that + was removed in Tcl 9.0. + + +.. class:: StringVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds a string. + The default value is ``''``. + + .. method:: get() + + Return the value of the variable as a :class:`str`. + + +.. class:: IntVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds an integer. + The default value is ``0``. + + .. method:: get() + + Return the value of the variable as an :class:`int`. + + +.. class:: DoubleVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds a float. + The default value is ``0.0``. + + .. method:: get() + + Return the value of the variable as a :class:`float`. + + +.. class:: BooleanVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds a boolean. + The default value is ``False``. + + .. method:: get() + + Return the value of the variable as a :class:`bool`. + Raise a :exc:`ValueError` if the value cannot be interpreted as a + boolean. + + .. method:: initialize(value) + :no-typesetting: + + .. method:: set(value) + + Set the variable to *value*, converting it to a boolean. + + .. versionadded:: 3.3 + The *initialize* spelling. + + +Image classes +^^^^^^^^^^^^^ + +.. class:: Image(imgtype, name=None, cnf={}, master=None, **kw) + + Base class for Tk images. + *imgtype* is the Tk image type, one of ``'photo'`` or ``'bitmap'``. + An image is a named object that can be displayed by widgets through their + *image* option; deleting all references to the :class:`!Image` object + deletes the underlying Tk image. + Usually you create a :class:`PhotoImage` or :class:`BitmapImage` rather than + an :class:`!Image` directly. + + The image's configuration options are given by *cnf* and *kw* and may be + queried and changed later with the mapping protocol (using ``image[key]``) + or with the :meth:`configure` method. + + .. method:: config(**kw) + :no-typesetting: + + .. method:: configure(**kw) + + Modify one or more configuration options of the image. + The valid options depend on the image type; see :class:`PhotoImage` and + :class:`BitmapImage`. + :meth:`config` is an alias of :meth:`!configure`. + + .. method:: height() + + Return the height of the image, in pixels. + + .. method:: width() + + Return the width of the image, in pixels. + + .. method:: type() + + Return the type of the image, that is the value of *imgtype* with which + it was created (for example ``'photo'`` or ``'bitmap'``). + + +.. class:: PhotoImage(name=None, cnf={}, master=None, **kw) + + A full-color image (the Tk ``photo`` image type), stored internally with a + varying degree of transparency per pixel. + It can read and write GIF, PPM/PGM and (in Tk 8.6 and later) PNG files, read + SVG files (in Tk 9.0 and later), and be drawn in widgets. + Inherits from :class:`Image`. + + The configuration options include *data* (the image contents as a string), + *file* (the name of a file to read the contents from), *format* (the name of + the file format handler), *width* and *height* (the size of the image, used + when building it up piece by piece), *gamma* and *palette*. + + .. method:: blank() + + Blank the image; that is, set the entire image to have no data, so that + it is displayed as transparent and the background of whatever window it + is displayed in shows through. + + .. method:: cget(option) + + Return the current value of the configuration option *option*. + + .. method:: copy(*, from_coords=None, zoom=None, subsample=None) + + Return a new :class:`PhotoImage` with a copy of this image. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + ``(x1, y1)`` and ``(x2, y2)`` specify diagonally opposite corners of the + rectangle. + If *x2* and *y2* are not specified, they default to the bottom-right + corner of the source image. + The pixels copied include the left and top edges of the rectangle but not + the bottom or right edges. + If *from_coords* is not given, the whole source image is copied. + + If *zoom* or *subsample* are specified, the image is transformed as in + the :meth:`zoom` or :meth:`subsample` methods. + The value must be a single integer or a pair of integers. + + .. versionchanged:: 3.13 + Added the *from_coords*, *zoom* and *subsample* parameters. + + + .. method:: copy_replace(sourceImage, *, from_coords=None, to=None, \ + shrink=False, zoom=None, subsample=None, \ + compositingrule=None) + + Copy a region from *sourceImage* (which must be a :class:`PhotoImage`) + into this image, possibly with pixel zooming and/or subsampling. + If no options are specified, the whole of *sourceImage* is copied into + this image, starting at coordinates ``(0, 0)``. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied, as in the :meth:`copy` method. + + *to* specifies a rectangular sub-region of the destination image to be + affected. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If *x2* and *y2* are not specified, they default to ``(x1, y1)`` plus the + size of the source region (after subsampling and zooming, if specified). + If *x2* and *y2* are specified, the source region is replicated if + necessary to fill the destination region in a tiled fashion. + + If *shrink* is true, the size of the destination image is reduced, if + necessary, so that the region being copied into is at the bottom-right + corner of the image. + + If *zoom* or *subsample* are specified, the image is transformed as in + the :meth:`zoom` or :meth:`subsample` methods. + The value must be a single integer or a pair of integers. + + *compositingrule* specifies how transparent pixels in the source image + are combined with the destination image. + With ``'overlay'`` (the default), the old contents of the destination + image remain visible, as if the source image were printed on a piece of + transparent film and placed over the top of the destination. + With ``'set'``, the old contents of the destination image are discarded + and the source image is used as-is. + + .. versionadded:: 3.13 + + + .. method:: data(format=None, *, from_coords=None, background=None, \ + grayscale=False) + + Return the image data. + + *format* specifies the name of the image file format handler to use. + If it is not given, the data is returned as a tuple (one element per row) + of strings containing space-separated (one element per pixel/column) + colors in ``#RRGGBB`` format. + + *from_coords* specifies a rectangular region of the image to be returned. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If only *x1* and *y1* are specified, the region extends from ``(x1, y1)`` + to the bottom-right corner of the image. + If all four coordinates are given, they specify diagonally opposite + corners of the region, including ``(x1, y1)`` and excluding ``(x2, y2)``. + If *from_coords* is not given, the whole image is returned. + + If *background* is specified, the data does not contain any transparency + information; in all transparent pixels the color is replaced by the + specified color. + + If *grayscale* is true, the data does not contain color information; all + pixel data is transformed into grayscale. + + .. versionadded:: 3.13 + + + .. method:: get(x, y) + + Return the color of the pixel at coordinates (*x*, *y*) as an + ``(r, g, b)`` tuple of three integers between 0 and 255, representing the + red, green and blue components respectively. + + .. method:: put(data, to=None) + + Set pixels of the image to the colors given in *data*, which must be a + string or a nested sequence of horizontal rows of pixel colors (for + example ``"{red green} {blue yellow}"``). + + *to* specifies the coordinates of the region of the image into which the + data are copied. + It must be a tuple or a list of 2 or 4 integers ``(x1, y1)`` or + ``(x1, y1, x2, y2)`` giving the top-left corner, and optionally the + bottom-right corner, of the region. + The default position is ``(0, 0)``. + + .. method:: read(filename, format=None, *, from_coords=None, to=None, \ + shrink=False) + + Read image data from the file named *filename* into the image. + + *format* specifies the format of the image data in the file. + + *from_coords* specifies a rectangular sub-region of the image file data + to be copied to the destination image. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If only *x1* and *y1* are specified, the region extends from ``(x1, y1)`` + to the bottom-right corner of the image in the file. + If all four coordinates are given, they specify diagonally opposite + corners of the region. + If *from_coords* is not given, the whole of the image in the file is + read. + + *to* specifies the coordinates of the top-left corner of the region of + the image into which the data are read. + The default is ``(0, 0)``. + + If *shrink* is true, the size of the image is reduced, if necessary, so + that the region into which the file data are read is at the bottom-right + corner of the image. + + .. versionadded:: 3.13 + + + .. method:: subsample(x, y='', *, from_coords=None) + + Return a new :class:`PhotoImage` based on this image but using only every + *x*-th pixel in the X direction and every *y*-th pixel in the Y + direction. + If *y* is not given, it defaults to the same value as *x*. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied, as in the :meth:`copy` method. + + .. versionchanged:: 3.13 + Added the *from_coords* parameter. + + + .. method:: transparency_get(x, y) + + Return ``True`` if the pixel at coordinates (*x*, *y*) is fully + transparent, ``False`` otherwise. + + .. versionadded:: 3.8 + + + .. method:: transparency_set(x, y, boolean) + + Make the pixel at coordinates (*x*, *y*) fully transparent if *boolean* + is true, fully opaque otherwise. + + .. versionadded:: 3.8 + + + .. method:: write(filename, format=None, from_coords=None, *, \ + background=None, grayscale=False) + + Write image data from the image to the file named *filename*. + + *format* specifies the name of the image file format handler to use. + If it is not given, the format is guessed from the file extension. + + *from_coords* specifies a rectangular region of the image to be written. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If only *x1* and *y1* are specified, the region extends from ``(x1, y1)`` + to the bottom-right corner of the image. + If all four coordinates are given, they specify diagonally opposite + corners of the region. + If *from_coords* is not given, the whole image is written. + + If *background* is specified, the data does not contain any transparency + information; in all transparent pixels the color is replaced by the + specified color. + + If *grayscale* is true, the data does not contain color information; all + pixel data is transformed into grayscale. + + .. versionchanged:: 3.13 + Added the *background* and *grayscale* parameters. + + + .. method:: zoom(x, y='', *, from_coords=None) + + Return a new :class:`PhotoImage` with this image magnified by a factor of + *x* in the X direction and *y* in the Y direction. + If *y* is not given, it defaults to the same value as *x*. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied, as in the :meth:`copy` method. + + .. versionchanged:: 3.13 + Added the *from_coords* parameter. + + + +.. class:: BitmapImage(name=None, cnf={}, master=None, **kw) + + A two-color image (the Tk ``bitmap`` image type) created from an X11 bitmap. + Each pixel displays a foreground color, a background color, or nothing + (producing a transparent effect). + Inherits from :class:`Image`. + + The configuration options are *data* or *file* (the source bitmap, given as + a string in X11 bitmap format or as the name of a file in that format), + *maskdata* or *maskfile* (the mask bitmap, in the same forms), and + *foreground* and *background* (the two colors). + For pixels where the mask is zero the image displays nothing; for other + pixels it displays the foreground color where the source is one and the + background color where the source is zero. + If *background* is set to an empty string, the background pixels are + transparent. + + :class:`!BitmapImage` has no methods of its own beyond those inherited from + :class:`Image`. + + +Other classes +^^^^^^^^^^^^^ + +.. class:: Event() + + A container for the attributes of an event passed to a callback bound with + :meth:`Misc.bind`. + An :class:`!Event` instance has the following attributes, each corresponding + to a field of the underlying Tk event; depending on the event type, some + attributes may be set to the string ``'??'`` to indicate that they are not + meaningful. + See :ref:`bindings-and-events`. + + .. attribute:: serial + + The serial number of the event. + + .. attribute:: num + + The mouse button that was pressed or released (for button events). + + .. attribute:: focus + + Whether the window has the focus (for ``Enter`` and ``Leave`` events). + + .. attribute:: height + width + + The new height and width of the window (for ``Configure`` and ``Expose`` + events). + + .. attribute:: keycode + + The keycode of the key that was pressed or released. + + .. attribute:: state + + The state of the event, as a number (for most events) or a string (for + ``Visibility`` events). + + .. attribute:: time + + The timestamp of the event, in milliseconds. + + .. attribute:: x + y + + The pointer position relative to the widget, in pixels. + + .. attribute:: x_root + y_root + + The pointer position relative to the top-left corner of the screen, in + pixels. + + .. attribute:: char + + The character typed, as a string (for key events). + + .. attribute:: send_event + + ``True`` if the event was sent by another application. + + .. attribute:: keysym + + The symbolic name of the key that was pressed or released. + + .. attribute:: keysym_num + + The numeric value of :attr:`keysym`. + + .. attribute:: type + + The :class:`EventType` of the event. + + .. attribute:: widget + + The widget on which the event occurred. + + .. attribute:: delta + + The amount the mouse wheel was rotated (for ``MouseWheel`` events). + + .. attribute:: user_data + + The data string of a virtual event, as passed to the *data* option of + :meth:`Misc.event_generate`. + It is ``'??'`` for non-virtual events. + + .. versionadded:: 3.15 + + .. attribute:: detail + + A fixed detail string for ``Enter``, ``Leave``, ``FocusIn``, ``FocusOut`` + and ``ConfigureRequest`` events (see the Tcl/Tk documentation). + It is ``'??'`` for other events. + + .. versionadded:: 3.15 + + +.. class:: EventType(*values) + + An :class:`enum.StrEnum` enumerating the Tk event types, used as the value + of :attr:`Event.type`. + Its members include, among others, ``KeyPress``, ``KeyRelease``, + ``ButtonPress``, ``ButtonRelease``, ``Motion``, ``Enter``, ``Leave``, + ``FocusIn``, ``FocusOut``, ``Configure``, ``Map``, ``Unmap``, ``Expose``, + ``Destroy`` and ``MouseWheel``. + + .. versionadded:: 3.6 + + + +.. class:: CallWrapper(func, subst, widget) + + Internal helper that wraps a Python callback so that it can be invoked from + Tcl. + *func* is the Python function, *subst* is an optional function that + pre-processes the Tcl arguments, and *widget* is the widget used for error + reporting. + Instances are created automatically by :meth:`Misc.register`; this class is + not normally used directly. + + +Module-level functions +^^^^^^^^^^^^^^^^^^^^^^ + +.. function:: NoDefaultRoot() + + Inhibit the creation of an implicit default root window. + Afterwards :mod:`!tkinter` no longer creates a shared default root + automatically, and operations that rely on one --- such as constructing a + widget without an explicit *master* --- raise a :exc:`RuntimeError`. + Call this early in larger applications to make the root window explicit. + +.. function:: mainloop(n=0) + + Run the Tk main event loop on the default root window until all windows are + destroyed. + Equivalent to calling :meth:`Misc.mainloop` on the default root. + +.. function:: getboolean(s) + + Convert the Tcl boolean string *s* (one of ``'1'``, ``'true'``, ``'yes'``, + ``'on'`` and similar, or their false counterparts) to a Python + :class:`bool`. + Raise :exc:`TclError` for an invalid value. + +.. function:: getdouble(s) + + Convert *s* to a floating-point number. + This is the built-in :class:`float`. + +.. function:: getint(s) + + Convert *s* to an integer. + This is the built-in :class:`int`. + +.. function:: image_names() + + Return the names of all existing images in the default root's interpreter. + +.. function:: image_types() + + Return the available image types (such as ``'photo'`` and ``'bitmap'``) in + the default root's interpreter. + +Constants +^^^^^^^^^ + +The following symbolic constants are available in both the :mod:`!tkinter` +and :mod:`!tkinter.constants` namespaces. + +.. data:: TRUE + YES + ON + + Truthy values, all equal to the integer ``1``. + +.. data:: FALSE + NO + OFF + + Falsy values, all equal to the integer ``0``. + +.. data:: N + S + E + W + NE + NW + SE + SW + NS + EW + NSEW + CENTER + + Compass directions (``'n'``, ``'s'``, ``'e'``, ``'w'`` and the diagonals and + edges) plus ``CENTER`` (``'center'``), used as values for the *anchor* and + *sticky* options and by methods such as :meth:`Misc.grid_anchor`. + +.. data:: LEFT + RIGHT + TOP + BOTTOM + + Sides for the *side* option of the packer (see :meth:`Pack.pack_configure`). + +.. data:: X + Y + BOTH + NONE + + Values for the *fill* option of the packer: ``'x'``, ``'y'``, ``'both'`` or + ``'none'``. + +.. data:: RAISED + SUNKEN + FLAT + RIDGE + GROOVE + SOLID + + Values for the *relief* option, which controls a widget's 3-D border. + +.. data:: HORIZONTAL + VERTICAL + + Values for the *orient* option of widgets such as :class:`Scale`, + :class:`Scrollbar` and :class:`PanedWindow`. + +.. data:: CHAR + WORD + + Values for the *wrap* option of the :class:`Text` widget, selecting line + wrapping on character or word boundaries. + +.. data:: BASELINE + + The text-alignment value ``'baseline'``. + +.. data:: INSIDE + OUTSIDE + + Values for the *bordermode* option of the placer (see + :meth:`Place.place_configure`). + +.. data:: INSERT + CURRENT + END + ANCHOR + SEL + SEL_FIRST + SEL_LAST + + Symbolic indices used by the :class:`Text`, :class:`Entry`, :class:`Listbox` + and :class:`Canvas` widgets, such as ``'insert'`` (the insertion cursor), + ``'current'``, ``'end'``, ``'anchor'`` and the bounds of the selection + (``'sel.first'`` and ``'sel.last'``). + +.. data:: ALL + + The special tag ``'all'``, which matches every item of a :class:`Canvas` or + every character of a :class:`Text` (for example ``canvas.delete(ALL)``). + +.. data:: NORMAL + DISABLED + ACTIVE + HIDDEN + + Values for the *state* option of various widgets and items. + +.. data:: CASCADE + CHECKBUTTON + COMMAND + RADIOBUTTON + SEPARATOR + + Menu entry types, used as the *itemType* argument of :meth:`Menu.add` and + :meth:`Menu.insert`. + +.. data:: SINGLE + BROWSE + MULTIPLE + EXTENDED + + Values for the *selectmode* option of the :class:`Listbox` widget. + +.. data:: PIESLICE + CHORD + ARC + + Values for the *style* option of :class:`Canvas` arc items. + +.. data:: BUTT + PROJECTING + ROUND + BEVEL + MITER + + Values for the *capstyle* (``'butt'``, ``'projecting'``, ``'round'``) and + *joinstyle* (``'round'``, ``'bevel'``, ``'miter'``) options of + :class:`Canvas` line items. + +.. data:: FIRST + LAST + + Values for the *arrow* option of :class:`Canvas` line items, indicating + which ends have arrowheads. + +.. data:: MOVETO + SCROLL + + The first argument passed by a :class:`Scrollbar` to the :meth:`XView.xview` + or :meth:`YView.yview` method of the scrolled widget. + +.. data:: UNITS + PAGES + + Values for the *what* argument of :meth:`XView.xview_scroll` and + :meth:`YView.yview_scroll`. + +.. data:: UNDERLINE + NUMERIC + DOTBOX + + Other option values: ``'underline'``, ``'numeric'`` and ``'dotbox'``. diff --git a/Doc/library/tkinter.scrolledtext.rst b/Doc/library/tkinter.scrolledtext.rst index 763e24929d74b51..30aef8748edb72b 100644 --- a/Doc/library/tkinter.scrolledtext.rst +++ b/Doc/library/tkinter.scrolledtext.rst @@ -1,25 +1,24 @@ -:mod:`!tkinter.scrolledtext` --- Scrolled Text Widget +:mod:`!tkinter.scrolledtext` --- Scrolled text widget ===================================================== .. module:: tkinter.scrolledtext - :platform: Tk :synopsis: Text widget with a vertical scroll bar. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/tkinter/scrolledtext.py` -------------- -The :mod:`tkinter.scrolledtext` module provides a class of the same name which +The :mod:`!tkinter.scrolledtext` module provides a class of the same name which implements a basic text widget which has a vertical scroll bar configured to do the "right thing." Using the :class:`ScrolledText` class is a lot easier than setting up a text widget and scroll bar directly. -The text widget and scrollbar are packed together in a :class:`Frame`, and the -methods of the :class:`Grid` and :class:`Pack` geometry managers are acquired -from the :class:`Frame` object. This allows the :class:`ScrolledText` widget to -be used directly to achieve most normal geometry management behavior. +The text widget and scrollbar are packed together in a :class:`~tkinter.Frame`, +and the methods of the :class:`~tkinter.Pack`, :class:`~tkinter.Grid` and +:class:`~tkinter.Place` geometry managers are acquired from the +:class:`~tkinter.Frame` object. +This allows the :class:`ScrolledText` widget to be used directly to achieve +most normal geometry management behavior. Should more specific control be necessary, the following attributes are available: diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index 628e9f945ac3658..9d90770a5840eed 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -4,23 +4,23 @@ .. module:: tkinter.ttk :synopsis: Tk themed widget set -.. sectionauthor:: Guilherme Polo - **Source code:** :source:`Lib/tkinter/ttk.py` .. index:: single: ttk -------------- -The :mod:`tkinter.ttk` module provides access to the Tk themed widget set, +The :mod:`!tkinter.ttk` module provides access to the Tk themed widget set, introduced in Tk 8.5. It provides additional benefits including anti-aliased font rendering under X11 and window transparency (requiring a composition window manager on X11). -The basic idea for :mod:`tkinter.ttk` is to separate, to the extent possible, +The basic idea for :mod:`!tkinter.ttk` is to separate, to the extent possible, the code implementing a widget's behavior from the code implementing its appearance. +.. versionadded:: 3.1 + .. seealso:: @@ -40,18 +40,18 @@ To override the basic Tk widgets, the import should follow the Tk import:: from tkinter import * from tkinter.ttk import * -That code causes several :mod:`tkinter.ttk` widgets (:class:`Button`, +That code causes several :mod:`!tkinter.ttk` widgets (:class:`Button`, :class:`Checkbutton`, :class:`Entry`, :class:`Frame`, :class:`Label`, :class:`LabelFrame`, :class:`Menubutton`, :class:`PanedWindow`, :class:`Radiobutton`, :class:`Scale` and :class:`Scrollbar`) to automatically replace the Tk widgets. -This has the direct benefit of using the new widgets which gives a better -look and feel across platforms; however, the replacement widgets are not -completely compatible. The main difference is that widget options such as -"fg", "bg" and others related to widget styling are no -longer present in Ttk widgets. Instead, use the :class:`ttk.Style` class -for improved styling effects. +This has the direct benefit of using the new widgets which gives a better look +and feel across platforms; however, the replacement widgets are not completely +compatible. +The main difference is that widget options such as "fg", "bg" and others +related to widget styling are no longer present in Ttk widgets. +Instead, use the :class:`ttk.Style {%- endblock -%} {% block body %}

{{ docstitle|e }}

@@ -21,63 +50,76 @@

{{ docstitle|e }}

{% trans %}Welcome! This is the official documentation for Python {{ release }}.{% endtrans %}

{% trans %}Documentation sections:{% endtrans %}

- - -
- - - - - - - - - - - - - -
+
+ + +
+ +

{% trans %}Other resources:{% endtrans %}

+
+ + +

{% trans %}Indices, glossary, and search:{% endtrans %}

- - -
- - - - - - -
+
+ + +

{% trans %}Project information:{% endtrans %}

- - -
- - - - - - - -
+ {% endblock %} diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 1cb0200822d9fe0..54c6eb9b5ef7e47 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -28,7 +28,7 @@ {% block extrahead %} {% if builder == "html" %} {% if enable_analytics %} - + {% endif %} {% if pagename == 'whatsnew/changelog' and not embedded %} diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 6a1611afadb57c1..2539edb51ba3a09 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -14,8 +14,7 @@ There are two variants of the interactive :term:`REPL`. The classic basic interpreter is supported on all platforms with minimal line control capabilities. -On Windows, or Unix-like systems with :mod:`curses` support, -a new interactive shell is used by default. +Since Python 3.13, a new interactive shell is used by default. This one supports color, multiline editing, history browsing, and paste mode. To disable color, see :ref:`using-on-controlling-color` for details. Function keys provide some additional functionality. diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 9d0fab8861d2a96..4f7da5253f78bc7 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -359,7 +359,7 @@ Usually, a method is called right after it is bound:: x.f() -In the :class:`!MyClass` example, this will return the string ``'hello world'``. +If ``x = MyClass()``, as above, this will return the string ``'hello world'``. However, it is not necessary to call a method right away: ``x.f`` is a method object, and can be stored away and called at a later time. For example:: @@ -420,7 +420,7 @@ of the class:: 'Buddy' As discussed in :ref:`tut-object`, shared data can have possibly surprising -effects with involving :term:`mutable` objects such as lists and dictionaries. +effects involving :term:`mutable` objects such as lists and dictionaries. For example, the *tricks* list in the following code should not be used as a class variable because just a single list would be shared by all *Dog* instances:: @@ -663,6 +663,9 @@ Taken together, these properties make it possible to design reliable and extensible classes with multiple inheritance. For more detail, see :ref:`python_2.3_mro`. +In some cases multiple inheritance is not allowed; see :ref:`multiple-inheritance` +for details. + .. _tut-private: @@ -926,6 +929,25 @@ Examples:: >>> list(data[i] for i in range(len(data)-1, -1, -1)) ['f', 'l', 'o', 'g'] + >>> x = [[1,2,3], [], [4, 5]] + >>> g = (*i for i in x) + >>> list(g) + [1, 2, 3, 4, 5] + +In most cases, generator expressions must be wrapped in parentheses. As a +special case, however, when provided as the sole argument to a function (as in +the examples involving ``sum``, ``set``, ``max``, and ``list`` above), the +generator expression does not need to be wrapped in an additional set of +parentheses. That is to say, the following two pieces of code are semantically +equivalent:: + + >>> f(x for x in y) + >>> f((x for x in y)) + +as are the following:: + + >>> f(*x for x in y) + >>> f((*x for x in y)) .. rubric:: Footnotes diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 5c0e8f34bf82f41..8bac8df4368c00a 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -155,8 +155,8 @@ that takes an iterable is :func:`sum`:: 6 Later we will see more functions that return iterables and take iterables as -arguments. In chapter :ref:`tut-structures`, we will discuss in more detail about -:func:`list`. +arguments. In chapter :ref:`tut-structures`, we will discuss :func:`list` in more +detail. .. _tut-break: @@ -251,6 +251,7 @@ statements: a ``try`` statement's ``else`` clause runs when no exception occurs, and a loop's ``else`` clause runs when no ``break`` occurs. For more on the ``try`` statement and exceptions, see :ref:`tut-handling`. +.. index:: single: ...; ellipsis literal .. _tut-pass: :keyword:`!pass` Statements @@ -277,6 +278,12 @@ at a more abstract level. The :keyword:`!pass` is silently ignored:: ... pass # Remember to implement this! ... +For this last case, many people use the ellipsis literal :code:`...` instead of +:code:`pass`. This use has no special meaning to Python, and is not part of +the language definition (you could use any constant expression here), but +:code:`...` is used conventionally as a placeholder body as well. +See :ref:`bltin-ellipsis-object`. + .. _tut-match: @@ -434,7 +441,7 @@ Several other key features of this statement: ``False`` and ``None`` are compared by identity. - Patterns may use named constants. These must be dotted names - to prevent them from being interpreted as capture variable:: + to prevent them from being interpreted as capture variables:: from enum import Enum class Color(Enum): @@ -561,7 +568,7 @@ This example, as usual, demonstrates some new Python features: Different types define different methods. Methods of different types may have the same name without causing ambiguity. (It is possible to define your own object types and methods, using *classes*, see :ref:`tut-classes`) - The method :meth:`!append` shown in the example is defined for list objects; it + The method :meth:`~list.append` shown in the example is defined for list objects; it adds a new element at the end of the list. In this example it is equivalent to ``result = result + [a]``, but more efficient. @@ -1032,31 +1039,28 @@ blank, visually separating the summary from the rest of the description. The following lines should be one or more paragraphs describing the object's calling conventions, its side effects, etc. -The Python parser does not strip indentation from multi-line string literals in -Python, so tools that process documentation have to strip indentation if -desired. This is done using the following convention. The first non-blank line -*after* the first line of the string determines the amount of indentation for -the entire documentation string. (We can't use the first line since it is -generally adjacent to the string's opening quotes so its indentation is not -apparent in the string literal.) Whitespace "equivalent" to this indentation is -then stripped from the start of all lines of the string. Lines that are -indented less should not occur, but if they occur all their leading whitespace -should be stripped. Equivalence of whitespace should be tested after expansion -of tabs (to 8 spaces, normally). +The Python parser strips indentation from multi-line string literals when they +serve as module, class, or function docstrings. Here is an example of a multi-line docstring:: >>> def my_function(): ... """Do nothing, but document it. ... - ... No, really, it doesn't do anything. + ... No, really, it doesn't do anything: + ... + ... >>> my_function() + ... >>> ... """ ... pass ... >>> print(my_function.__doc__) Do nothing, but document it. - No, really, it doesn't do anything. + No, really, it doesn't do anything: + + >>> my_function() + >>> .. _tut-annotations: @@ -1064,7 +1068,6 @@ Here is an example of a multi-line docstring:: Function Annotations -------------------- -.. sectionauthor:: Zachary Ware .. index:: pair: function; annotations single: ->; function annotations @@ -1074,7 +1077,7 @@ Function Annotations information about the types used by user-defined functions (see :pep:`3107` and :pep:`484` for more information). -:term:`Annotations ` are stored in the :attr:`!__annotations__` +:term:`Annotations ` are stored in the :attr:`~object.__annotations__` attribute of the function as a dictionary and have no effect on any other part of the function. Parameter annotations are defined by a colon after the parameter name, followed by an expression evaluating to the value of the annotation. Return annotations are @@ -1098,12 +1101,11 @@ value annotated:: Intermezzo: Coding Style ======================== -.. sectionauthor:: Georg Brandl .. index:: pair: coding; style Now that you are about to write longer, more complex pieces of Python, it is a good time to talk about *coding style*. Most languages can be written (or more -concise, *formatted*) in different styles; some are more readable than others. +concisely, *formatted*) in different styles; some are more readable than others. Making it easy for others to read your code is always a good idea, and adopting a nice coding style helps tremendously for that. diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index cbe780e075baf59..276e31a3056f0ee 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -12,24 +12,23 @@ and adds some new things as well. More on Lists ============= -The list data type has some more methods. Here are all of the methods of list -objects: +The :ref:`list ` data type has some more methods. Here are all +of the methods of list objects: - -.. method:: list.append(x) +.. method:: list.append(value, /) :noindex: Add an item to the end of the list. Similar to ``a[len(a):] = [x]``. -.. method:: list.extend(iterable) +.. method:: list.extend(iterable, /) :noindex: Extend the list by appending all the items from the iterable. Similar to ``a[len(a):] = iterable``. -.. method:: list.insert(i, x) +.. method:: list.insert(index, value, /) :noindex: Insert an item at a given position. The first argument is the index of the @@ -37,14 +36,14 @@ objects: the list, and ``a.insert(len(a), x)`` is equivalent to ``a.append(x)``. -.. method:: list.remove(x) +.. method:: list.remove(value, /) :noindex: - Remove the first item from the list whose value is equal to *x*. It raises a + Remove the first item from the list whose value is equal to *value*. It raises a :exc:`ValueError` if there is no such item. -.. method:: list.pop([i]) +.. method:: list.pop(index=-1, /) :noindex: Remove the item at the given position in the list, and return it. If no index @@ -59,10 +58,10 @@ objects: Remove all items from the list. Similar to ``del a[:]``. -.. method:: list.index(x[, start[, end]]) +.. method:: list.index(value[, start[, stop]]) :noindex: - Return zero-based index in the list of the first item whose value is equal to *x*. + Return zero-based index of the first occurrence of *value* in the list. Raises a :exc:`ValueError` if there is no such item. The optional arguments *start* and *end* are interpreted as in the slice @@ -71,10 +70,10 @@ objects: sequence rather than the *start* argument. -.. method:: list.count(x) +.. method:: list.count(value, /) :noindex: - Return the number of times *x* appears in the list. + Return the number of times *value* appears in the list. .. method:: list.sort(*, key=None, reverse=False) @@ -137,13 +136,10 @@ comparison. Using Lists as Stacks --------------------- -.. sectionauthor:: Ka-Ping Yee - - The list methods make it very easy to use a list as a stack, where the last element added is the first element retrieved ("last-in, first-out"). To add an -item to the top of the stack, use :meth:`!append`. To retrieve an item from the -top of the stack, use :meth:`!pop` without an explicit index. For example:: +item to the top of the stack, use :meth:`~list.append`. To retrieve an item from the +top of the stack, use :meth:`~list.pop` without an explicit index. For example:: >>> stack = [3, 4, 5] >>> stack.append(6) @@ -167,8 +163,6 @@ top of the stack, use :meth:`!pop` without an explicit index. For example:: Using Lists as Queues --------------------- -.. sectionauthor:: Ka-Ping Yee - It is also possible to use a list as a queue, where the first element added is the first element retrieved ("first-in, first-out"); however, lists are not efficient for this purpose. While appends and pops from the end of list are @@ -334,13 +328,54 @@ The :func:`zip` function would do a great job for this use case:: See :ref:`tut-unpacking-arguments` for details on the asterisk in this line. +Unpacking in Lists and List Comprehensions +------------------------------------------ + +The section on :ref:`tut-unpacking-arguments` describes the use of ``*`` to +"unpack" the elements of an iterable object, providing each one separately as +an argument to a function. Unpacking can also be used in other contexts, for +example, when creating lists. When specifying elements of a list, prefixing an +expression by a ``*`` will unpack the result of that expression, adding each of +its elements to the list we're creating:: + + >>> x = [1, 2, 3] + >>> [0, *x, 4, 5, 6] + [0, 1, 2, 3, 4, 5, 6] + +This only works if the expression following the ``*`` evaluates to an iterable +object; trying to unpack a non-iterable object will raise an exception:: + + >>> x = 1 + >>> [0, *x, 2, 3, 4] + Traceback (most recent call last): + File "", line 1, in + [0, *x, 2, 3, 4] + TypeError: Value after * must be an iterable, not int + +Unpacking can also be used in list comprehensions, as a way to build a new list +representing the concatenation of an arbitrary number of iterables:: + + >>> x = [[1, 2, 3], [4, 5, 6], [], [7], [8, 9]] + >>> [*element for element in x] + [1, 2, 3, 4, 5, 6, 7, 8, 9] + +Note that the effect is that each element from ``x`` is unpacked. This works +for arbitrary iterable objects, not just lists:: + + >>> x = [[1, 2, 3], 'cat', {'spam': 'eggs'}] + >>> [*element for element in x] + [1, 2, 3, 'c', 'a', 't', 'spam'] + +But if the objects in ``x`` are not iterable, this expression would again raise +an exception. + .. _tut-del: The :keyword:`!del` statement ============================= There is a way to remove an item from a list given its index instead of its -value: the :keyword:`del` statement. This differs from the :meth:`!pop` method +value: the :keyword:`del` statement. This differs from the :meth:`~list.pop` method which returns a value. The :keyword:`!del` statement can also be used to remove slices from a list or clear the entire list (which we did earlier by assignment of an empty list to the slice). For example:: @@ -395,7 +430,10 @@ A tuple consists of a number of values separated by commas, for instance:: >>> v = ([1, 2, 3], [3, 2, 1]) >>> v ([1, 2, 3], [3, 2, 1]) - + >>> # they support unpacking just like lists: + >>> x = [1, 2, 3] + >>> 0, *x, 4 + (0, 1, 2, 3, 4) As you see, on output tuples are always enclosed in parentheses, so that nested tuples are interpreted correctly; they may be input with or without surrounding @@ -445,15 +483,19 @@ packing and sequence unpacking. Sets ==== -Python also includes a data type for *sets*. A set is an unordered collection -with no duplicate elements. Basic uses include membership testing and -eliminating duplicate entries. Set objects also support mathematical operations -like union, intersection, difference, and symmetric difference. +Python also includes a data type for :ref:`sets `. A set is +an unordered collection with no duplicate elements. Basic uses include +membership testing and eliminating duplicate entries. Set objects also +support mathematical operations like union, intersection, difference, and +symmetric difference. Curly braces or the :func:`set` function can be used to create sets. Note: to create an empty set you have to use ``set()``, not ``{}``; the latter creates an empty dictionary, a data structure that we discuss in the next section. +Because sets are unordered, iterating over them or printing them can +produce the elements in a different order than you expect. + Here is a brief demonstration:: >>> basket = {'apple', 'orange', 'apple', 'pear', 'orange', 'banana'} @@ -480,12 +522,16 @@ Here is a brief demonstration:: {'r', 'd', 'b', 'm', 'z', 'l'} Similarly to :ref:`list comprehensions `, set comprehensions -are also supported:: +are also supported, including comprehensions with unpacking:: >>> a = {x for x in 'abracadabra' if x not in 'abc'} >>> a {'r', 'd'} + >>> fruits = [{'apple', 'avocado', 'apricot'}, {'banana', 'blueberry'}] + >>> {*fruit for fruit in fruits} + {'blueberry', 'banana', 'avocado', 'apple', 'apricot'} + .. _tut-dictionaries: @@ -500,8 +546,8 @@ any immutable type; strings and numbers can always be keys. Tuples can be used as keys if they contain only strings, numbers, or tuples; if a tuple contains any mutable object either directly or indirectly, it cannot be used as a key. You can't use lists as keys, since lists can be modified in place using index -assignments, slice assignments, or methods like :meth:`!append` and -:meth:`!extend`. +assignments, slice assignments, or methods like :meth:`~list.append` and +:meth:`~list.extend`. It is best to think of a dictionary as a set of *key: value* pairs, with the requirement that the keys are unique (within one dictionary). A pair of @@ -512,8 +558,12 @@ dictionary; this is also the way dictionaries are written on output. The main operations on a dictionary are storing a value with some key and extracting the value given the key. It is also possible to delete a key:value pair with ``del``. If you store using a key that is already in use, the old -value associated with that key is forgotten. It is an error to extract a value -using a non-existent key. +value associated with that key is forgotten. + +Extracting a value for a non-existent key by subscripting (``d[key]``) raises a +:exc:`KeyError`. To avoid getting this error when trying to access a possibly +non-existent key, use the :meth:`~dict.get` method instead, which returns +``None`` (or a specified default value) if the key is not in the dictionary. Performing ``list(d)`` on a dictionary returns a list of all the keys used in the dictionary, in insertion order (if you want it sorted, just use @@ -528,6 +578,12 @@ Here is a small example using a dictionary:: {'jack': 4098, 'sape': 4139, 'guido': 4127} >>> tel['jack'] 4098 + >>> tel['irv'] + Traceback (most recent call last): + File "", line 1, in + KeyError: 'irv' + >>> print(tel.get('irv')) + None >>> del tel['sape'] >>> tel['irv'] = 4127 >>> tel @@ -553,6 +609,18 @@ arbitrary key and value expressions:: >>> {x: x**2 for x in (2, 4, 6)} {2: 4, 4: 16, 6: 36} +And dictionary unpacking (via ``**``) can be used to merge multiple +dictionaries:: + + >>> odds = {i: i**2 for i in (1, 3, 5)} + >>> evens = {i: i**2 for i in (2, 4, 6)} + >>> {**odds, **evens} + {1: 1, 3: 9, 5: 25, 2: 4, 4: 16, 6: 36} + + >>> all_values = [odds, evens, {0: 0}] + >>> {**i for i in all_values} + {1: 1, 3: 9, 5: 25, 2: 4, 4: 16, 6: 36, 0: 0} + When the keys are simple strings, it is sometimes easier to specify pairs using keyword arguments:: diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 1c20fa2f0b6ae58..3c6edf2c4793abd 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -121,9 +121,9 @@ A :keyword:`try` statement may have more than one *except clause*, to specify handlers for different exceptions. At most one handler will be executed. Handlers only handle exceptions that occur in the corresponding *try clause*, not in other handlers of the same :keyword:`!try` statement. An *except clause* -may name multiple exceptions as a parenthesized tuple, for example:: +may name multiple exceptions, for example:: - ... except (RuntimeError, TypeError, NameError): + ... except RuntimeError, TypeError, NameError: ... pass A class in an :keyword:`except` clause matches exceptions which are instances of the @@ -549,9 +549,9 @@ caught like any other exception. :: >>> try: ... f() ... except Exception as e: - ... print(f'caught {type(e)}: e') + ... print(f'caught {type(e)}: {e}') ... - caught : e + caught : there were problems (2 sub-exceptions) >>> By using ``except*`` instead of ``except``, we can selectively diff --git a/Doc/tutorial/floatingpoint.rst b/Doc/tutorial/floatingpoint.rst index dfe2d1d3a8378fa..37e23ba1cd0c812 100644 --- a/Doc/tutorial/floatingpoint.rst +++ b/Doc/tutorial/floatingpoint.rst @@ -9,10 +9,6 @@ Floating-Point Arithmetic: Issues and Limitations ************************************************** -.. sectionauthor:: Tim Peters -.. sectionauthor:: Raymond Hettinger - - Floating-point numbers are represented in computer hardware as base 2 (binary) fractions. For example, the **decimal** fraction ``0.625`` has value 6/10 + 2/100 + 5/1000, and in the same way the **binary** fraction ``0.101`` diff --git a/Doc/tutorial/index.rst b/Doc/tutorial/index.rst index d0bf77dc40d0a13..20fe161be4acc26 100644 --- a/Doc/tutorial/index.rst +++ b/Doc/tutorial/index.rst @@ -15,7 +15,7 @@ together with its interpreted nature, make it an ideal language for scripting and rapid application development in many areas on most platforms. The Python interpreter and the extensive standard library are freely available -in source or binary form for all major platforms from the Python web site, +in source or binary form for all major platforms from the Python website, https://www.python.org/, and may be freely distributed. The same site also contains distributions of and pointers to many free third party Python modules, programs and tools, and additional documentation. diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index ea546c6a29df443..a00f06cf46c41a3 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -95,11 +95,11 @@ Some examples:: >>> repr((x, y, ('spam', 'eggs'))) "(32.5, 40000, ('spam', 'eggs'))" -The :mod:`string` module also contains support for so-called -:ref:`$-strings ` that offer yet another way to -substitute values into strings, using placeholders like ``$x`` and replacing -them with values from a dictionary. This syntax is easy to use, although -it offers much less control of the formatting. +The :mod:`string` module contains support for a simple templating approach +based upon regular expressions, via :class:`string.Template`. +This offers yet another way to substitute values into strings, +using placeholders like ``$x`` and replacing them with values from a dictionary. +This syntax is easy to use, although it offers much less control for formatting. .. index:: single: formatted string literal diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index cd52607142485e6..45a2ca21801437f 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -16,7 +16,7 @@ Unix shell's search path makes it possible to start it by typing the command: .. code-block:: text - python3.15 + python3.16 to the shell. [#]_ Since the choice of the directory where the interpreter lives is an installation option, other places are possible; check with your local @@ -34,13 +34,13 @@ status. If that doesn't work, you can exit the interpreter by typing the following command: ``quit()``. The interpreter's line-editing features include interactive editing, history -substitution and code completion on systems that support the `GNU Readline -`_ library. +substitution and code completion on most systems. Perhaps the quickest check to see whether command line editing is supported is -typing :kbd:`Control-P` to the first Python prompt you get. If it beeps, you -have command line editing; see Appendix :ref:`tut-interacting` for an -introduction to the keys. If nothing appears to happen, or if ``^P`` is -echoed, command line editing isn't available; you'll only be able to use +typing a word in on the Python prompt, then pressing Left arrow (or :kbd:`Control-b`). +If the cursor moves, you have command line editing; see Appendix +:ref:`tut-interacting` for an introduction to the keys. +If nothing appears to happen, or if a sequence like ``^[[D`` or ``^B`` appears, +command line editing isn't available; you'll only be able to use backspace to remove characters from the current line. The interpreter operates somewhat like the Unix shell: when called with standard @@ -97,8 +97,8 @@ before printing the first prompt: .. code-block:: shell-session - $ python3.15 - Python 3.15 (default, May 7 2025, 15:46:04) + $ python3.16 + Python 3.16 (default, May 7 2026, 19:03:04) [GCC 10.2.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 9e06e03991bc968..7778e37a9adaa95 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -49,7 +49,7 @@ primary prompt, ``>>>``. (It shouldn't take long.) Numbers ------- -The interpreter acts as a simple calculator: you can type an expression at it +The interpreter acts as a simple calculator: you can type an expression into it and it will write the value. Expression syntax is straightforward: the operators ``+``, ``-``, ``*`` and ``/`` can be used to perform arithmetic; parentheses (``()``) can be used for grouping. @@ -184,11 +184,11 @@ If you don't want characters prefaced by ``\`` to be interpreted as special characters, you can use *raw strings* by adding an ``r`` before the first quote:: - >>> print('C:\some\name') # here \n means newline! - C:\some + >>> print('C:\this\name') # here \t means tab, \n means newline + C: his ame - >>> print(r'C:\some\name') # note the r before the quote - C:\some\name + >>> print(r'C:\this\name') # note the r before the quote + C:\this\name There is one subtle aspect to raw strings: a raw string may not end in an odd number of ``\`` characters; see @@ -420,7 +420,7 @@ type, i.e. it is possible to change their content:: [1, 8, 27, 64, 125] You can also add new items at the end of the list, by using -the :meth:`!list.append` *method* (we will see more about methods later):: +the :meth:`list.append` *method* (we will see more about methods later):: >>> cubes.append(216) # add the cube of 6 >>> cubes.append(7 ** 3) # and the cube of 7 diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index d83ecca270b1602..dec2008add1bf50 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -1,13 +1,13 @@ .. _tut-brieftour: ********************************** -Brief Tour of the Standard Library +Brief tour of the standard library ********************************** .. _tut-os-interface: -Operating System Interface +Operating system interface ========================== The :mod:`os` module provides dozens of functions for interacting with the @@ -15,7 +15,7 @@ operating system:: >>> import os >>> os.getcwd() # Return the current working directory - 'C:\\Python315' + 'C:\\Python316' >>> os.chdir('/server/accesslogs') # Change current working directory >>> os.system('mkdir today') # Run the command mkdir in the system shell 0 @@ -47,7 +47,7 @@ a higher level interface that is easier to use:: .. _tut-file-wildcards: -File Wildcards +File wildcards ============== The :mod:`glob` module provides a function for making file lists from directory @@ -60,7 +60,7 @@ wildcard searches:: .. _tut-command-line-arguments: -Command Line Arguments +Command-line arguments ====================== Common utility scripts often need to process command line arguments. These @@ -97,7 +97,7 @@ to ``['alpha.txt', 'beta.txt']``. .. _tut-stderr: -Error Output Redirection and Program Termination +Error output redirection and program termination ================================================ The :mod:`sys` module also has attributes for *stdin*, *stdout*, and *stderr*. @@ -112,7 +112,7 @@ The most direct way to terminate a script is to use ``sys.exit()``. .. _tut-string-pattern-matching: -String Pattern Matching +String pattern matching ======================= The :mod:`re` module provides regular expression tools for advanced string @@ -175,7 +175,7 @@ computations. .. _tut-internet-access: -Internet Access +Internet access =============== There are a number of modules for accessing the internet and processing internet @@ -183,13 +183,13 @@ protocols. Two of the simplest are :mod:`urllib.request` for retrieving data from URLs and :mod:`smtplib` for sending mail:: >>> from urllib.request import urlopen - >>> with urlopen('http://worldtimeapi.org/api/timezone/etc/UTC.txt') as response: + >>> with urlopen('https://docs.python.org/3/') as response: ... for line in response: ... line = line.decode() # Convert bytes to a str - ... if line.startswith('datetime'): + ... if 'updated' in line: ... print(line.rstrip()) # Remove trailing newline ... - datetime: 2022-01-01T01:36:47.689215+00:00 + Last updated on Nov 11, 2025 (20:11 UTC). >>> import smtplib >>> server = smtplib.SMTP('localhost') @@ -206,7 +206,7 @@ from URLs and :mod:`smtplib` for sending mail:: .. _tut-dates-and-times: -Dates and Times +Dates and times =============== The :mod:`datetime` module supplies classes for manipulating dates and times in @@ -216,15 +216,15 @@ formatting and manipulation. The module also supports objects that are timezone aware. :: >>> # dates are easily constructed and formatted - >>> from datetime import date - >>> now = date.today() + >>> import datetime as dt + >>> now = dt.date.today() >>> now datetime.date(2003, 12, 2) >>> now.strftime("%m-%d-%y. %d %b %Y is a %A on the %d day of %B.") '12-02-03. 02 Dec 2003 is a Tuesday on the 02 day of December.' >>> # dates support calendar arithmetic - >>> birthday = date(1964, 7, 31) + >>> birthday = dt.date(1964, 7, 31) >>> age = now - birthday >>> age.days 14368 @@ -232,7 +232,7 @@ aware. :: .. _tut-data-compression: -Data Compression +Data compression ================ Common data archiving and compression formats are directly supported by modules @@ -254,7 +254,7 @@ including: :mod:`zlib`, :mod:`gzip`, :mod:`bz2`, :mod:`lzma`, :mod:`zipfile` and .. _tut-performance-measurement: -Performance Measurement +Performance measurement ======================= Some Python users develop a deep interest in knowing the relative performance of @@ -278,7 +278,7 @@ larger blocks of code. .. _tut-quality-control: -Quality Control +Quality control =============== One approach for developing high quality software is to write tests for each @@ -324,7 +324,7 @@ file:: .. _tut-batteries-included: -Batteries Included +Batteries included ================== Python has a "batteries included" philosophy. This is best seen through the @@ -335,7 +335,7 @@ sophisticated and robust capabilities of its larger packages. For example: names, no direct knowledge or handling of XML is needed. * The :mod:`email` package is a library for managing email messages, including - MIME and other :rfc:`2822`-based message documents. Unlike :mod:`smtplib` and + MIME and other :rfc:`5322`-based message documents. Unlike :mod:`smtplib` and :mod:`poplib` which actually send and receive messages, the email package has a complete toolset for building or decoding complex message structures (including attachments) and for implementing internet encoding and header diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 678b71c9274c1c6..cf3a44850b0b8f2 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -1,7 +1,7 @@ .. _tut-brieftourtwo: ********************************************** -Brief Tour of the Standard Library --- Part II +Brief tour of the standard library --- part II ********************************************** This second tour covers more advanced modules that support professional @@ -10,7 +10,7 @@ programming needs. These modules rarely occur in small scripts. .. _tut-output-formatting: -Output Formatting +Output formatting ================= The :mod:`reprlib` module provides a version of :func:`repr` customized for @@ -130,7 +130,7 @@ templates for XML files, plain text reports, and HTML web reports. .. _tut-binary-formats: -Working with Binary Data Record Layouts +Working with binary data record layouts ======================================= The :mod:`struct` module provides :func:`~struct.pack` and @@ -178,14 +178,13 @@ tasks in background while the main program continues to run:: class AsyncZip(threading.Thread): def __init__(self, infile, outfile): - threading.Thread.__init__(self) + super().__init__() self.infile = infile self.outfile = outfile def run(self): - f = zipfile.ZipFile(self.outfile, 'w', zipfile.ZIP_DEFLATED) - f.write(self.infile) - f.close() + with zipfile.ZipFile(self.outfile, 'w', zipfile.ZIP_DEFLATED) as f: + f.write(self.infile) print('Finished background zip of:', self.infile) background = AsyncZip('mydata.txt', 'myarchive.zip') @@ -245,7 +244,7 @@ application. .. _tut-weak-references: -Weak References +Weak references =============== Python does automatic memory management (reference counting for most objects and @@ -279,14 +278,14 @@ applications include caching objects that are expensive to create:: Traceback (most recent call last): File "", line 1, in d['primary'] # entry was automatically removed - File "C:/python315/lib/weakref.py", line 46, in __getitem__ + File "C:/python316/lib/weakref.py", line 46, in __getitem__ o = self.data[key]() KeyError: 'primary' .. _tut-list-tools: -Tools for Working with Lists +Tools for working with lists ============================ Many data structure needs can be met with the built-in list type. However, @@ -352,7 +351,7 @@ not want to run a full list sort:: .. _tut-decimal-fp: -Decimal Floating-Point Arithmetic +Decimal floating-point arithmetic ================================= The :mod:`decimal` module offers a :class:`~decimal.Decimal` datatype for diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index f362e1943b666f7..6b6b8a768e3f58d 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -88,6 +88,11 @@ For example: '~/envs/tutorial-env/lib/python3.5/site-packages'] >>> +Note that the activated virtual environment does not alter the ``PYTHONPATH`` variable in any way. +This may lead to unexpected results if the path includes references to code which is incompatible with +the Python version the virtual environment is using. The best practice is to ``unset PYTHONPATH`` +in bash or the equivalent for the shell you are using. + To deactivate a virtual environment, type:: deactivate diff --git a/Doc/tutorial/whatnow.rst b/Doc/tutorial/whatnow.rst index dbe2d7fc09927e2..aae8f29b0077627 100644 --- a/Doc/tutorial/whatnow.rst +++ b/Doc/tutorial/whatnow.rst @@ -30,7 +30,7 @@ the set are: More Python resources: -* https://www.python.org: The major Python web site. It contains code, +* https://www.python.org: The major Python website. It contains code, documentation, and pointers to Python-related pages around the web. * https://docs.python.org: Fast access to Python's documentation. @@ -68,6 +68,6 @@ already contain the solution for your problem. .. rubric:: Footnotes -.. [#] "Cheese Shop" is a Monty Python's sketch: a customer enters a cheese shop, +.. [#] "Cheese Shop" is a Monty Python sketch: a customer enters a cheese shop, but whatever cheese he asks for, the clerk says it's missing. diff --git a/Doc/using/android.rst b/Doc/using/android.rst index cb762310328f1cb..60a135693181412 100644 --- a/Doc/using/android.rst +++ b/Doc/using/android.rst @@ -30,7 +30,7 @@ Adding Python to an Android app Most app developers should use one of the following tools, which will provide a much easier experience: -* `Briefcase `__, from the BeeWare project +* `Briefcase `__, from the BeeWare project * `Buildozer `__, from the Kivy project * `Chaquopy `__ * `pyqtdeploy `__ @@ -40,8 +40,15 @@ If you're sure you want to do all of this manually, read on. You can use the :source:`testbed app ` as a guide; each step below contains a link to the relevant file. -* Build Python by following the instructions in :source:`Android/README.md`. - This will create the directory ``cross-build/HOST/prefix``. +* First, acquire a build of Python for Android: + + * The easiest way is to download an Android release from `python.org + `__. The ``prefix`` directory + mentioned below is at the top level of the package. + + * Or if you want to build it yourself, follow the instructions in + :source:`Android/README.md`. The ``prefix`` directory will be created under + :samp:`cross-build/{HOST}`. * Add code to your :source:`build.gradle ` file to copy the following items into your project. All except your own Python diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index cad49e2deeb46f8..677fbbae3f4219a 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -49,9 +49,9 @@ additional methods of invocation: appropriately named script from that directory. * When called with ``-c command``, it executes the Python statement(s) given as *command*. Here *command* may contain multiple statements separated by - newlines. Leading whitespace is significant in Python statements! -* When called with ``-m module-name``, the given module is located on the - Python module path and executed as a script. + newlines. +* When called with ``-m module-name``, the given module is located using the standard + import mechanism and executed as a script. In non-interactive mode, the entire input is parsed before it is executed. @@ -78,8 +78,8 @@ source. .. option:: -m - Search :data:`sys.path` for the named module and execute its contents as - the :mod:`__main__` module. + Locate the module using the standard import mechanism and execute its contents + as the :mod:`__main__` module. Since the argument is a *module* name, you must not give a file extension (``.py``). The module name should be a valid absolute Python module name, but @@ -254,6 +254,15 @@ Miscellaneous options .. versionchanged:: 3.5 Affects also comparisons of :class:`bytes` with :class:`int`. + .. deprecated:: 3.15 + + Deprecate :option:`-b` and :option:`!-bb` command line options + and schedule them to become no-op in Python 3.17. + These were primarily helpers for the Python 2 -> 3 transition. + Starting with Python 3.17, no :exc:`BytesWarning` will be raised + for these cases; use a type checker instead. + + .. option:: -B If given, Python won't try to write ``.pyc`` files on the @@ -369,8 +378,8 @@ Miscellaneous options .. option:: -R Turn on hash randomization. This option only has an effect if the - :envvar:`PYTHONHASHSEED` environment variable is set to ``0``, since hash - randomization is enabled by default. + :envvar:`PYTHONHASHSEED` environment variable is set to anything other + than ``random``, since hash randomization is enabled by default. On previous versions of Python, this option turns on hash randomization, so that the :meth:`~object.__hash__` values of str and bytes objects @@ -381,7 +390,7 @@ Miscellaneous options Hash randomization is intended to provide protection against a denial-of-service caused by carefully chosen inputs that exploit the worst case performance of a dict construction, *O*\ (*n*\ :sup:`2`) complexity. See - http://ocert.org/advisories/ocert-2011-003.html for details. + https://ocert.org/advisories/ocert-2011-003.html for details. :envvar:`PYTHONHASHSEED` allows you to set a fixed value for the hash seed secret. @@ -470,8 +479,10 @@ Miscellaneous options The *action* field is as explained above but only applies to warnings that match the remaining fields. - The *message* field must match the whole warning message; this match is - case-insensitive. + The *message* field must match the start of the warning message; + this match is case-insensitive. + If it starts and ends with a forward slash (``/``), it specifies + a regular expression, otherwise it specifies a literal string. The *category* field matches the warning category (ex: ``DeprecationWarning``). This must be a class name; the match test @@ -480,6 +491,10 @@ Miscellaneous options The *module* field matches the (fully qualified) module name; this match is case-sensitive. + If it starts and ends with a forward slash (``/``), it specifies + a regular expression that the start of the fully qualified module name + must match, otherwise it specifies a literal string that the fully + qualified module name must be equal to. The *lineno* field matches the line number, where zero matches all line numbers and is thus equivalent to an omitted line number. @@ -497,6 +512,9 @@ Miscellaneous options See :ref:`warning-filter` and :ref:`describing-warning-filters` for more details. + .. versionchanged:: 3.15 + Added regular expression support for *message* and *module*. + .. option:: -x @@ -636,13 +654,17 @@ Miscellaneous options .. versionadded:: 3.13 - * :samp:`-X presite={package.module}` specifies a module that should be - imported before the :mod:`site` module is executed and before the + * :samp:`-X presite={module}` or :samp:`-X presite={module:func}` specifies + an entry point that should be executed before the :mod:`site` module is + executed and before the :mod:`__main__` module exists. Therefore, the imported module isn't :mod:`__main__`. This can be used to execute code early during Python initialization. Python needs to be :ref:`built in debug mode ` for this option to exist. See also :envvar:`PYTHON_PRESITE`. + .. versionchanged:: 3.15 + Accept also ``module:func`` entry point format. + .. versionadded:: 3.13 * :samp:`-X gil={0,1}` forces the GIL to be disabled or enabled, @@ -669,6 +691,13 @@ Miscellaneous options .. versionadded:: 3.14 + * :samp:`-X pathconfig_warnings={0,1}` if true (``1``) then + :ref:`sys-path-init` is allowed to log warnings into stderr. + If false (``0``) suppress these warnings. Set to true by default. + See also :envvar:`PYTHON_PATHCONFIG_WARNINGS`. + + .. versionadded:: 3.15 + * :samp:`-X tlbc={0,1}` enables (1, the default) or disables (0) thread-local bytecode in builds configured with :option:`--disable-gil`. When disabled, this also disables the specializing interpreter. See also @@ -676,6 +705,13 @@ Miscellaneous options .. versionadded:: 3.14 + * :samp:`-X lazy_imports={all,normal}` controls lazy import behavior. + ``all`` makes all imports lazy by default, and ``normal`` (the default) + respects the ``lazy`` keyword in source code. + See also :envvar:`PYTHON_LAZY_IMPORTS`. + + .. versionadded:: 3.15 + It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -923,8 +959,9 @@ conflict. .. envvar:: PYTHONNOUSERSITE - If this is set, Python won't add the :data:`user site-packages directory - ` to :data:`sys.path`. + This is equivalent to the :option:`-s` option. If this is set, Python won't + add the :data:`user site-packages directory ` to + :data:`sys.path`. .. seealso:: @@ -938,6 +975,9 @@ conflict. and :ref:`installation paths ` for ``python -m pip install --user``. + To disable the user site-packages, see :envvar:`PYTHONNOUSERSITE` or the :option:`-s` + option. + .. seealso:: :pep:`370` -- Per user site-packages directory @@ -971,6 +1011,9 @@ conflict. See :ref:`warning-filter` and :ref:`describing-warning-filters` for more details. + .. versionchanged:: 3.15 + Added regular expression support for *message* and *module*. + .. envvar:: PYTHONFAULTHANDLER @@ -1045,6 +1088,13 @@ conflict. * ``pymalloc_debug``: same as ``pymalloc`` but also install debug hooks. * ``mimalloc_debug``: same as ``mimalloc`` but also install debug hooks. + .. note:: + + In the :term:`free-threaded ` build, the ``malloc``, + ``malloc_debug``, ``pymalloc``, and ``pymalloc_debug`` values are not + supported. Only ``default``, ``debug``, ``mimalloc``, and + ``mimalloc_debug`` are accepted. + .. versionadded:: 3.6 .. versionchanged:: 3.7 @@ -1054,18 +1104,48 @@ conflict. .. envvar:: PYTHONMALLOCSTATS If set to a non-empty string, Python will print statistics of the - :ref:`pymalloc memory allocator ` every time a new pymalloc object - arena is created, and on shutdown. + :ref:`pymalloc memory allocator ` or the + :ref:`mimalloc memory allocator ` (whichever is in use) + every time a new object arena is created, and on shutdown. This variable is ignored if the :envvar:`PYTHONMALLOC` environment variable is used to force the :c:func:`malloc` allocator of the C library, or if - Python is configured without ``pymalloc`` support. + Python is configured without both ``pymalloc`` and ``mimalloc`` support. .. versionchanged:: 3.6 This variable can now also be used on Python compiled in release mode. It now has no effect if set to an empty string. +.. envvar:: PYTHON_PYMALLOC_HUGEPAGES + + If set to a non-zero integer, enable huge page support for + :ref:`pymalloc ` arenas. Set to ``0`` or unset to disable. + Python must be compiled with :option:`--with-pymalloc-hugepages` for this + variable to have any effect. + + When enabled, arena allocation uses ``MAP_HUGETLB`` (Linux) or + ``MEM_LARGE_PAGES`` (Windows) with automatic fallback to regular pages if + huge pages are not available. + + .. warning:: + + On Linux, if the huge-page pool is exhausted, page faults — including + copy-on-write faults triggered by :func:`os.fork` — deliver ``SIGBUS`` + and kill the process. Only enable this in environments where the + huge-page pool is properly sized and fork-safety is not a concern. + + On Windows you need a special privilege. See the + `Windows documentation for large pages + `_ + for details. Python will fail on startup if the required privilege + `SeLockMemoryPrivilege + `_ + is not held by the user. + + .. versionadded:: 3.15 + + .. envvar:: PYTHONLEGACYWINDOWSFSENCODING If set to a non-empty string, the default :term:`filesystem encoding and @@ -1073,9 +1153,6 @@ conflict. 'replace', respectively. Otherwise, the new defaults 'utf-8' and 'surrogatepass' are used. - This may also be enabled at runtime with - :func:`sys._enablelegacywindowsfsencoding`. - .. availability:: Windows. .. versionadded:: 3.6 @@ -1256,12 +1333,18 @@ conflict. .. envvar:: PYTHON_BASIC_REPL If this variable is set to any value, the interpreter will not attempt to - load the Python-based :term:`REPL` that requires :mod:`curses` and - :mod:`readline`, and will instead use the traditional parser-based - :term:`REPL`. + load the Python-based :term:`REPL` that requires :mod:`readline`, and will + instead use the traditional parser-based :term:`REPL`. .. versionadded:: 3.13 +.. envvar:: PYTHON_BASIC_COMPLETER + + If this variable is set to any value, PyREPL will use :mod:`rlcompleter` to + implement tab completion, instead of the default one which uses colors. + + .. versionadded:: 3.15 + .. envvar:: PYTHON_HISTORY This environment variable can be used to set the location of a @@ -1301,6 +1384,14 @@ conflict. .. versionadded:: 3.14 +.. envvar:: PYTHON_PATHCONFIG_WARNINGS + + If true (``1``) then :ref:`sys-path-init` is allowed to log warnings into + stderr. If false (``0``) suppress these warnings. Set to true by default. + See also :option:`-X pathconfig_warnings<-X>`. + + .. versionadded:: 3.15 + .. envvar:: PYTHON_JIT On builds where experimental just-in-time compilation is available, this @@ -1319,6 +1410,16 @@ conflict. .. versionadded:: 3.14 +.. envvar:: PYTHON_LAZY_IMPORTS + + Controls lazy import behavior. Accepts two values: ``all`` makes all + imports lazy by default, and ``normal`` (the default) respects the + ``lazy`` keyword in source code. + + See also the :option:`-X lazy_imports <-X>` command-line option. + + .. versionadded:: 3.15 + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ @@ -1356,4 +1457,7 @@ Debug-mode variables Needs Python configured with the :option:`--with-pydebug` build option. + .. versionchanged:: 3.15 + Accept also ``module:func`` entry point format. + .. versionadded:: 3.13 diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 2cda9587975ddcd..22a10db976c4fc6 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -4,10 +4,13 @@ Configure Python .. highlight:: sh + +.. _build-requirements: + Build Requirements ================== -Features and minimum versions required to build CPython: +To build CPython, you will need: * A `C11 `_ compiler. `Optional C11 features @@ -22,50 +25,135 @@ Features and minimum versions required to build CPython: * Support for threads. -* OpenSSL 1.1.1 is the minimum version and OpenSSL 3.0.16 is the recommended - minimum version for the :mod:`ssl` and :mod:`hashlib` extension modules. +.. versionchanged:: 3.5 + On Windows, Visual Studio 2015 or later is now required. + +.. versionchanged:: 3.6 + Selected C99 features, like ```` and ``static inline`` functions, + are now required. -* SQLite 3.15.2 for the :mod:`sqlite3` extension module. +.. versionchanged:: 3.7 + Thread support is now required. -* Tcl/Tk 8.5.12 for the :mod:`tkinter` module. +.. versionchanged:: 3.11 + C11 compiler, IEEE 754 and NaN support are now required. + On Windows, Visual Studio 2017 or later is required. -* `libmpdec `_ 2.5.0 - for the :mod:`decimal` module. +See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform +support". -* Autoconf 2.72 and aclocal 1.16.5 are required to regenerate the - :file:`configure` script. + +.. _optional-module-requirements: + +Requirements for optional modules +--------------------------------- + +Some :term:`optional modules ` of the standard library +require third-party libraries installed for development +(for example, header files must be available). + +Missing requirements are reported in the ``configure`` output. +Modules that are missing due to missing dependencies are listed near the end +of the ``make`` output, +sometimes using an internal name, for example, ``_ctypes`` for :mod:`ctypes` +module. + +If you distribute a CPython interpreter without optional modules, +it's best practice to advise users, who generally expect that +standard library modules are available. + +Dependencies to build optional modules are: + +.. list-table:: + :header-rows: 1 + :align: left + + * - Dependency + - Minimum version + - Python module + * - `libbz2 `_ + - + - :mod:`bz2` + * - `libffi `_ + - 3.3.0 recommended + - :mod:`ctypes` + * - `liblzma `_ + - + - :mod:`lzma` + * - `libmpdec `_ + - 2.5.0 + - :mod:`decimal` [1]_ + * - `libreadline `_ or + `libedit `_ [2]_ + - + - :mod:`readline` + * - `libuuid `_ + - + - ``_uuid`` [3]_ + * - `ncurses `_ [4]_ + - + - :mod:`curses` + * - `OpenSSL `_ + - | 3.0.18 recommended + | (1.1.1 minimum) + - :mod:`ssl`, :mod:`hashlib` [5]_ + * - `SQLite `_ + - 3.15.2 + - :mod:`sqlite3` + * - `Tcl/Tk `_ + - 8.5.12 + - :mod:`tkinter`, :ref:`IDLE `, :mod:`turtle` + * - `zlib `_ + - 1.2.2.1 + - :mod:`zlib`, :mod:`gzip`, :mod:`ensurepip` + * - `zstd `_ + - 1.4.5 + - :mod:`compression.zstd` + +.. [1] If *libmpdec* is not available, the :mod:`decimal` module will use + a pure-Python implementation. +.. [2] See :option:`--with-readline` for choosing the backend for the + :mod:`readline` module. +.. [3] The :mod:`uuid` module uses ``_uuid`` to generate "safe" UUIDs. + See the module documentation for details. +.. [4] The :mod:`curses` module requires the ``libncurses`` or ``libncursesw`` + library. + The :mod:`curses.panel` module additionally requires the ``libpanel`` or + ``libpanelw`` library. +.. [5] If OpenSSL is not available, the :mod:`hashlib` module will use + bundled implementations of several hash functions. + See :option:`--with-builtin-hashlib-hashes` for *forcing* usage of OpenSSL. + +Note that the table does not include all optional modules; in particular, +platform-specific modules like :mod:`winreg` are not listed here. + +.. seealso:: + + * The `devguide `_ + includes a full list of dependencies required to build all modules and + instructions on how to install them on common platforms. + * :option:`--with-system-expat` allows building with an external + `libexpat `_ library. + * :ref:`configure-options-for-dependencies` .. versionchanged:: 3.1 - Tcl/Tk version 8.3.1 is now required. + Tcl/Tk version 8.3.1 is now required for :mod:`tkinter`. .. versionchanged:: 3.5 - On Windows, Visual Studio 2015 or later is now required. - Tcl/Tk version 8.4 is now required. - -.. versionchanged:: 3.6 - Selected C99 features are now required, like ```` and ``static - inline`` functions. + Tcl/Tk version 8.4 is now required for :mod:`tkinter`. .. versionchanged:: 3.7 - Thread support and OpenSSL 1.0.2 are now required. + OpenSSL 1.0.2 is now required for :mod:`hashlib` and :mod:`ssl`. .. versionchanged:: 3.10 - OpenSSL 1.1.1 is now required. - Require SQLite 3.7.15. + OpenSSL 1.1.1 is now required for :mod:`hashlib` and :mod:`ssl`. + SQLite 3.7.15 is now required for :mod:`sqlite3`. .. versionchanged:: 3.11 - C11 compiler, IEEE 754 and NaN support are now required. - On Windows, Visual Studio 2017 or later is required. - Tcl/Tk version 8.5.12 is now required for the :mod:`tkinter` module. + Tcl/Tk version 8.5.12 is now required for :mod:`tkinter`. .. versionchanged:: 3.13 - Autoconf 2.71, aclocal 1.16.5 and SQLite 3.15.2 are now required. - -.. versionchanged:: 3.14 - Autoconf 2.72 is now required. - -See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform -support". + SQLite 3.15.2 is now required for :mod:`sqlite3`. Generated files @@ -94,8 +182,19 @@ The container is optional, the following command can be run locally:: autoreconf -ivf -Werror -The generated files can change depending on the exact ``autoconf-archive``, -``aclocal`` and ``pkg-config`` versions. +The generated files can change depending on the exact versions of the +tools used. +The container that CPython uses has +`Autoconf `_ 2.72, +``aclocal`` from `Automake `_ 1.16.5, +and `pkg-config `_ 1.8.1. + +.. versionchanged:: 3.13 + Autoconf 2.71 and aclocal 1.16.5 and are now used to regenerate + :file:`configure`. + +.. versionchanged:: 3.14 + Autoconf 2.72 is now used to regenerate :file:`configure`. .. _configure-options: @@ -222,6 +321,30 @@ General Options .. versionadded:: 3.11 +.. option:: --with-missing-stdlib-config=FILE + + Path to a `JSON `_ configuration file + containing custom error messages for missing :term:`standard library` modules. + + This option is intended for Python distributors who wish to provide + distribution-specific guidance when users encounter standard library + modules that are missing or packaged separately. + + The JSON file should map missing module names to custom error message strings. + For example, if your distribution packages :mod:`tkinter` and + :mod:`_tkinter` separately and excludes :mod:`!_gdbm` for legal reasons, + the configuration could contain: + + .. code-block:: json + + { + "_gdbm": "The '_gdbm' module is not available in this distribution", + "tkinter": "Install the python-tk package to use tkinter", + "_tkinter": "Install the python-tk package to use tkinter", + } + + .. versionadded:: 3.15 + .. option:: --enable-pystats Turn on internal Python performance statistics gathering. @@ -293,8 +416,11 @@ General Options .. option:: --disable-gil + .. c:macro:: Py_GIL_DISABLED + :no-typesetting: + Enables support for running Python without the :term:`global interpreter - lock` (GIL): free threading build. + lock` (GIL): :term:`free-threaded build`. Defines the ``Py_GIL_DISABLED`` macro and adds ``"t"`` to :data:`sys.abiflags`. @@ -336,6 +462,26 @@ General Options ``pkg-config`` options. +.. option:: --disable-epoll + + Build without ``epoll``, meaning that :py:func:`select.epoll` will not be + present even if the system provides an + :manpage:`epoll_create ` function. + This may be used on systems where :manpage:`!epoll_create` or + :manpage:`epoll_create1 ` is available + but incompatible with Linux semantics. + + .. versionadded:: 3.15 + +.. option:: --with-build-details-suffix=[yes|SUFFIX] + + Rename ``build-details.json`` to permit multiple co-located Python + installs. If a custom ``SUFFIX`` is supplied it is used verbatim, + otherwise one will be generated from the ``MULTIARCH`` tag with + ``-free-threading`` and ``-debug``, as appropriate. + + .. versionadded:: 3.16 + C compiler options ------------------ @@ -373,6 +519,8 @@ Linker options Name for machine-dependent library files. +.. _configure-options-for-dependencies: + Options for third-party dependencies ------------------------------------ @@ -395,12 +543,6 @@ Options for third-party dependencies C compiler and linker flags for ``gdbm``. -.. option:: LIBB2_CFLAGS -.. option:: LIBB2_LIBS - - C compiler and linker flags for ``libb2`` (:ref:`BLAKE2 `), - used by :mod:`hashlib` module, overriding ``pkg-config``. - .. option:: LIBEDIT_CFLAGS .. option:: LIBEDIT_LIBS @@ -419,11 +561,6 @@ Options for third-party dependencies C compiler and linker flags for ``libmpdec``, used by :mod:`decimal` module, overriding ``pkg-config``. - .. note:: - - These environment variables have no effect unless - :option:`--with-system-libmpdec` is specified. - .. option:: LIBLZMA_CFLAGS .. option:: LIBLZMA_LIBS @@ -646,11 +783,41 @@ also be used to improve performance. .. versionadded:: 3.14 +.. option:: --without-frame-pointers + + Disable frame pointers, which are enabled by default (see :pep:`831`). + + By default, the build appends flags to generate frame or backchain + pointers to ``BASECFLAGS``: + + - ``-fno-omit-frame-pointer`` and/or ``-mno-omit-leaf-frame-pointer`` + are added when the compiler supports them. + - ``-marm`` and/or ``-mno-thumb`` is added on 32-bit ARM when supported, + - on s390x platforms, when supported, ``-mbackchain`` is added *instead*. + of the above frame pointer flags. + - on ppc64le platforms, no compiler flags is needed since the power ABI + requires that compilers maintain a back chain by default. + + Frame pointers enable profilers, debuggers, and system tracing tools + (``perf``, ``eBPF``, ``dtrace``, ``gdb``) to walk the C call stack + without DWARF metadata. The flags propagate to third-party C + extensions through :mod:`sysconfig`. On compilers that do not + understand them, the build silently skips them. + + Downstream packagers and authors of native libraries built with + custom build systems should set the same flags so the unwind chain + stays unbroken across all native frames. + + .. versionadded:: 3.15 + .. option:: --without-mimalloc Disable the fast :ref:`mimalloc ` allocator (enabled by default). + This option cannot be used together with :option:`--disable-gil` + because the :term:`free-threaded ` build requires mimalloc. + See also :envvar:`PYTHONMALLOC` environment variable. .. option:: --without-pymalloc @@ -660,6 +827,36 @@ also be used to improve performance. See also :envvar:`PYTHONMALLOC` environment variable. +.. option:: --with-pymalloc-hugepages + + Enable huge page support for :ref:`pymalloc ` arenas (disabled by + default). When enabled, the arena size on 64-bit platforms is increased to + 2 MiB and arena allocation uses ``MAP_HUGETLB`` (Linux) or + ``MEM_LARGE_PAGES`` (Windows) with automatic fallback to regular pages. + + Even when compiled with this option, huge pages are **not** used at runtime + unless the :envvar:`PYTHON_PYMALLOC_HUGEPAGES` environment variable is set + to ``1``. This opt-in is required because huge pages + + * carry risks on Linux: if the huge-page pool is exhausted, page faults + (including copy-on-write faults after :func:`os.fork`) deliver ``SIGBUS`` + and kill the process. + + * need a special privilege on Windows. See the `Windows documentation for large pages + `_ + for details. Python will fail on startup if the required privilege + `SeLockMemoryPrivilege + `_ + is not held by the user. + + The configure script checks that the platform supports ``MAP_HUGETLB`` + and emits a warning if it is not available. + + On Windows, use the ``--pymalloc-hugepages`` flag with ``build.bat`` or + set the ``UsePymallocHugepages`` MSBuild property. + + .. versionadded:: 3.15 + .. option:: --without-doc-strings Disable static documentation strings to reduce the memory footprint (enabled @@ -739,9 +936,11 @@ See also the :ref:`Python Development Mode ` and the :option:`--with-trace-refs` configure option. .. versionchanged:: 3.8 - Release builds and debug builds are now ABI compatible: defining the + Release builds are now ABI compatible with debug builds: defining the ``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro (see the - :option:`--with-trace-refs` option). + :option:`--with-trace-refs` option). However, debug builds still expose + more symbols than release builds and code built against a debug build is not + necessarily compatible with a release build. Debug options @@ -843,6 +1042,21 @@ Linker options .. versionadded:: 3.10 +.. option:: --enable-static-libpython-for-interpreter + + Do not link the Python interpreter binary (``python3``) against the + shared Python library; instead, statically link the interpreter + against ``libpython`` as if ``--enable-shared`` had not been used, + but continue to build the shared ``libpython`` (for use by other + programs). + + This option does nothing if ``--enable-shared`` is not used. + + The default (when ``-enable-shared`` is used) is to link the Python + interpreter against the built shared library. + + .. versionadded:: 3.15 + Libraries options ----------------- @@ -856,22 +1070,6 @@ Libraries options Build the :mod:`!pyexpat` module using an installed ``expat`` library (default is no). -.. option:: --with-system-libmpdec - - Build the ``_decimal`` extension module using an installed ``mpdecimal`` - library, see the :mod:`decimal` module (default is yes). - - .. versionadded:: 3.3 - - .. versionchanged:: 3.13 - Default to using the installed ``mpdecimal`` library. - - .. deprecated-removed:: 3.13 3.15 - A copy of the ``mpdecimal`` library sources will no longer be distributed - with Python 3.15. - - .. seealso:: :option:`LIBMPDEC_CFLAGS` and :option:`LIBMPDEC_LIBS`. - .. option:: --with-readline=readline|editline Designate a backend library for the :mod:`readline` module. @@ -1398,6 +1596,12 @@ Compiler flags .. versionadded:: 3.7 +.. envvar:: CFLAGS_CEVAL + + Flags used to compile ``Python/ceval.c``. + + .. versionadded:: 3.14.5 + .. envvar:: CCSHARED Compiler flags used to build a shared library. @@ -1499,6 +1703,9 @@ Linker flags value to be able to build extension modules using the directories specified in the environment variables. + Please consider using ``EXE_LDFLAGS`` if the supplied linker flags are + executable specific, e.g. GCC's ``-pie`` flag. + .. envvar:: LIBS Linker flags to pass libraries to the linker when linking the Python @@ -1534,6 +1741,30 @@ Linker flags .. versionadded:: 3.8 +.. envvar:: EXE_LDFLAGS + + Linker flags used for building executable targets such as the + interpreter. If supplied, :envvar:`PY_CORE_EXE_LDFLAGS` + will be used in replacement of :envvar:`PY_CORE_LDFLAGS`. + + .. versionadded:: 3.15 + +.. envvar:: CONFIGURE_EXE_LDFLAGS + + Value of :envvar:`EXE_LDFLAGS` variable passed to the ``./configure`` + script. + + .. versionadded:: 3.15 + +.. envvar:: PY_CORE_EXE_LDFLAGS + + Linker flags used for building the interpreter and + executable targets. + + Default: ``$(PY_CORE_LDFLAGS)`` + + .. versionadded:: 3.15 + .. rubric:: Footnotes diff --git a/Doc/using/ios.rst b/Doc/using/ios.rst index 0fb28f8c866b02c..5e4033fb6cec7a0 100644 --- a/Doc/using/ios.rst +++ b/Doc/using/ios.rst @@ -170,7 +170,7 @@ helpful. To add Python to an iOS Xcode project: 1. Build or obtain a Python ``XCFramework``. See the instructions in - :source:`iOS/README.rst` (in the CPython source distribution) for details on + :source:`Apple/iOS/README.md` (in the CPython source distribution) for details on how to build a Python ``XCFramework``. At a minimum, you will need a build that supports ``arm64-apple-ios``, plus one of either ``arm64-apple-ios-simulator`` or ``x86_64-apple-ios-simulator``. @@ -180,22 +180,19 @@ To add Python to an iOS Xcode project: of your project; however, you can use any other location that you want by adjusting paths as needed. -3. Drag the ``iOS/Resources/dylib-Info-template.plist`` file into your project, - and ensure it is associated with the app target. - -4. Add your application code as a folder in your Xcode project. In the +3. Add your application code as a folder in your Xcode project. In the following instructions, we'll assume that your user code is in a folder named ``app`` in the root of your project; you can use any other location by adjusting paths as needed. Ensure that this folder is associated with your app target. -5. Select the app target by selecting the root node of your Xcode project, then +4. Select the app target by selecting the root node of your Xcode project, then the target name in the sidebar that appears. -6. In the "General" settings, under "Frameworks, Libraries and Embedded +5. In the "General" settings, under "Frameworks, Libraries and Embedded Content", add ``Python.xcframework``, with "Embed & Sign" selected. -7. In the "Build Settings" tab, modify the following: +6. In the "Build Settings" tab, modify the following: - Build Options @@ -211,86 +208,24 @@ To add Python to an iOS Xcode project: * Quoted Include In Framework Header: No -8. Add a build step that copies the Python standard library into your app. In - the "Build Phases" tab, add a new "Run Script" build step *before* the - "Embed Frameworks" step, but *after* the "Copy Bundle Resources" step. Name - the step "Install Target Specific Python Standard Library", disable the - "Based on dependency analysis" checkbox, and set the script content to: +7. Add a build step that processes the Python standard library, and your own + Python binary dependencies. In the "Build Phases" tab, add a new "Run + Script" build step *before* the "Embed Frameworks" step, but *after* the + "Copy Bundle Resources" step. Name the step "Process Python libraries", + disable the "Based on dependency analysis" checkbox, and set the script + content to: .. code-block:: bash - set -e - - mkdir -p "$CODESIGNING_FOLDER_PATH/python/lib" - if [ "$EFFECTIVE_PLATFORM_NAME" = "-iphonesimulator" ]; then - echo "Installing Python modules for iOS Simulator" - rsync -au --delete "$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" - else - echo "Installing Python modules for iOS Device" - rsync -au --delete "$PROJECT_DIR/Python.xcframework/ios-arm64/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" - fi - - Note that the name of the simulator "slice" in the XCframework may be - different, depending the CPU architectures your ``XCFramework`` supports. - -9. Add a second build step that processes the binary extension modules in the - standard library into "Framework" format. Add a "Run Script" build step - *directly after* the one you added in step 8, named "Prepare Python Binary - Modules". It should also have "Based on dependency analysis" unchecked, with - the following script content: + set -e + source $PROJECT_DIR/Python.xcframework/build/build_utils.sh + install_python Python.xcframework app - .. code-block:: bash + If you have placed your XCframework somewhere other than the root of your + project, modify the path to the first argument. - set -e - - install_dylib () { - INSTALL_BASE=$1 - FULL_EXT=$2 - - # The name of the extension file - EXT=$(basename "$FULL_EXT") - # The location of the extension file, relative to the bundle - RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} - # The path to the extension file, relative to the install base - PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/} - # The full dotted name of the extension module, constructed from the file path. - FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d "." -f 1 | tr "/" "."); - # A bundle identifier; not actually used, but required by Xcode framework packaging - FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr "_" "-") - # The name of the framework folder. - FRAMEWORK_FOLDER="Frameworks/$FULL_MODULE_NAME.framework" - - # If the framework folder doesn't exist, create it. - if [ ! -d "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" ]; then - echo "Creating framework for $RELATIVE_EXT" - mkdir -p "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" - cp "$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" - plutil -replace CFBundleExecutable -string "$FULL_MODULE_NAME" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" - plutil -replace CFBundleIdentifier -string "$FRAMEWORK_BUNDLE_ID" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" - fi - - echo "Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME" - mv "$FULL_EXT" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" - # Create a placeholder .fwork file where the .so was - echo "$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" > ${FULL_EXT%.so}.fwork - # Create a back reference to the .so file location in the framework - echo "${RELATIVE_EXT%.so}.fwork" > "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin" - } - - PYTHON_VER=$(ls -1 "$CODESIGNING_FOLDER_PATH/python/lib") - echo "Install Python $PYTHON_VER standard library extension modules..." - find "$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload" -name "*.so" | while read FULL_EXT; do - install_dylib python/lib/$PYTHON_VER/lib-dynload/ "$FULL_EXT" - done - - # Clean up dylib template - rm -f "$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist" - - echo "Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)..." - find "$CODESIGNING_FOLDER_PATH/Frameworks" -name "*.framework" -exec /usr/bin/codesign --force --sign "$EXPANDED_CODE_SIGN_IDENTITY" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der "{}" \; - -10. Add Objective C code to initialize and use a Python interpreter in embedded - mode. You should ensure that: +8. Add Objective C code to initialize and use a Python interpreter in embedded + mode. You should ensure that: * UTF-8 mode (:c:member:`PyPreConfig.utf8_mode`) is *enabled*; * Buffered stdio (:c:member:`PyConfig.buffered_stdio`) is *disabled*; @@ -309,22 +244,19 @@ To add Python to an iOS Xcode project: Your app's bundle location can be determined using ``[[NSBundle mainBundle] resourcePath]``. -Steps 8, 9 and 10 of these instructions assume that you have a single folder of +Steps 7 and 8 of these instructions assume that you have a single folder of pure Python application code, named ``app``. If you have third-party binary modules in your app, some additional steps will be required: * You need to ensure that any folders containing third-party binaries are - either associated with the app target, or copied in as part of step 8. Step 8 - should also purge any binaries that are not appropriate for the platform a - specific build is targeting (i.e., delete any device binaries if you're - building an app targeting the simulator). - -* Any folders that contain third-party binaries must be processed into - framework form by step 9. The invocation of ``install_dylib`` that processes - the ``lib-dynload`` folder can be copied and adapted for this purpose. + either associated with the app target, or are explicitly copied as part of + step 7. Step 7 should also purge any binaries that are not appropriate for + the platform a specific build is targeting (i.e., delete any device binaries + if you're building an app targeting the simulator). -* If you're using a separate folder for third-party packages, ensure that folder - is included as part of the :envvar:`PYTHONPATH` configuration in step 10. +* If you're using a separate folder for third-party packages, ensure that + folder is added to the end of the call to ``install_python`` in step 7, and + as part of the :envvar:`PYTHONPATH` configuration in step 8. * If any of the folders that contain third-party packages will contain ``.pth`` files, you should add that folder as a *site directory* (using @@ -334,25 +266,30 @@ modules in your app, some additional steps will be required: Testing a Python package ------------------------ -The CPython source tree contains :source:`a testbed project ` that +The CPython source tree contains :source:`a testbed project ` that is used to run the CPython test suite on the iOS simulator. This testbed can also be used as a testbed project for running your Python library's test suite on iOS. -After building or obtaining an iOS XCFramework (See :source:`iOS/README.rst` -for details), create a clone of the Python iOS testbed project by running: +After building or obtaining an iOS XCFramework (see :source:`Apple/iOS/README.md` +for details), create a clone of the Python iOS testbed project. If you used the +``Apple`` build script to build the XCframework, you can run: + +.. code-block:: bash + + $ python cross-build/iOS/testbed clone --app --app app-testbed + +Or, if you've sourced your own XCframework, by running: .. code-block:: bash - $ python iOS/testbed clone --framework --app --app app-testbed + $ python Apple/testbed clone --platform iOS --framework --app --app app-testbed -You will need to modify the ``iOS/testbed`` reference to point to that -directory in the CPython source tree; any folders specified with the ``--app`` -flag will be copied into the cloned testbed project. The resulting testbed will -be created in the ``app-testbed`` folder. In this example, the ``module1`` and -``module2`` would be importable modules at runtime. If your project has -additional dependencies, they can be installed into the -``app-testbed/iOSTestbed/app_packages`` folder (using ``pip install --target -app-testbed/iOSTestbed/app_packages`` or similar). +Any folders specified with the ``--app`` flag will be copied into the cloned +testbed project. The resulting testbed will be created in the ``app-testbed`` +folder. In this example, the ``module1`` and ``module2`` would be importable +modules at runtime. If your project has additional dependencies, they can be +installed into the ``app-testbed/Testbed/app_packages`` folder (using ``pip +install --target app-testbed/Testbed/app_packages`` or similar). You can then use the ``app-testbed`` folder to run the test suite for your app, For example, if ``module1.tests`` was the entry point to your test suite, you @@ -374,13 +311,29 @@ You can also open the testbed project in Xcode by running: This will allow you to use the full Xcode suite of tools for debugging. +The arguments used to run the test suite are defined as part of the test plan. +To modify the test plan, select the test plan node of the project tree (it +should be the first child of the root node), and select the "Configurations" +tab. Modify the "Arguments Passed On Launch" value to change the testing +arguments. + +The test plan also disables parallel testing, and specifies the use of the +``Testbed.lldbinit`` file for providing configuration of the debugger. The +default debugger configuration disables automatic breakpoints on the +``SIGINT``, ``SIGUSR1``, ``SIGUSR2``, and ``SIGXFSZ`` signals. + App Store Compliance ==================== The only mechanism for distributing apps to third-party iOS devices is to submit the app to the iOS App Store; apps submitted for distribution must pass Apple's app review process. This process includes a set of automated validation -rules that inspect the submitted application bundle for problematic code. +rules that inspect the submitted application bundle for problematic code. There +are some steps that must be taken to ensure that your app will be able to pass +these validation steps. + +Incompatible code in the standard library +----------------------------------------- The Python standard library contains some code that is known to violate these automated rules. While these violations appear to be false positives, Apple's @@ -391,3 +344,18 @@ The Python source tree contains :source:`a patch file ` that will remove all code that is known to cause issues with the App Store review process. This patch is applied automatically when building for iOS. + +Privacy manifests +----------------- + +In April 2025, Apple introduced a requirement for `certain third-party +libraries to provide a Privacy Manifest +`__. +As a result, if you have a binary module that uses one of the affected +libraries, you must provide an ``.xcprivacy`` file for that library. +OpenSSL is one library affected by this requirement, but there are others. + +If you produce a binary module named ``mymodule.so``, and use you the Xcode +build script described in step 7 above, you can place a ``mymodule.xcprivacy`` +file next to ``mymodule.so``, and the privacy manifest will be installed into +the required location when the binary module is converted into a framework. diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index f88f3c2e0785e49..6cf945de5b3f3b4 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -5,9 +5,6 @@ Using Python on macOS ********************* -.. sectionauthor:: Bob Savage -.. sectionauthor:: Ned Deily - This document aims to give an overview of macOS-specific behavior you should know about to get started with Python on Mac computers. Python on a Mac running macOS is very similar to Python on other Unix-derived platforms, @@ -39,7 +36,7 @@ Current installers provide a `universal2 binary `_ build of Python which runs natively on all Macs (Apple Silicon and Intel) that are supported by a wide range of macOS versions, -currently typically from at least **macOS 10.13 High Sierra** on. +currently typically from at least **macOS 10.15 Catalina** on. The downloaded file is a standard macOS installer package file (``.pkg``). File integrity information (checksum, size, sigstore signature, etc) for each file is included diff --git a/Doc/using/mac_installer_07_applications.png b/Doc/using/mac_installer_07_applications.png index 940219cad6f61c0..c8b3aa1099adfc9 100644 Binary files a/Doc/using/mac_installer_07_applications.png and b/Doc/using/mac_installer_07_applications.png differ diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index 9ec4e3419321a4e..829bdfe8b1121fd 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -6,9 +6,6 @@ Using Python on Unix platforms ******************************** -.. sectionauthor:: Shriphani Palakodety - - Getting and installing the latest version of Python =================================================== @@ -84,11 +81,17 @@ On FreeBSD and OpenBSD Building Python =============== +.. seealso:: + + If you want to contribute to CPython, refer to the + `devguide `_, + which includes build instructions and other tips on setting up environment. + If you want to compile CPython yourself, first thing you should do is get the `source `_. You can download either the -latest release's source or just grab a fresh `clone -`_. (If you want -to contribute patches, you will need a clone.) +latest release's source or grab a fresh `clone +`_. +You will also need to install the :ref:`build requirements `. The build process consists of the usual commands:: diff --git a/Doc/using/win_install_freethreaded.png b/Doc/using/win_install_freethreaded.png index 0aa01c1df6e051b..12b89c0165d994d 100644 Binary files a/Doc/using/win_install_freethreaded.png and b/Doc/using/win_install_freethreaded.png differ diff --git a/Doc/using/win_installer.png b/Doc/using/win_installer.png index 03bf2d7b16c599b..fc9605a79cfcc51 100644 Binary files a/Doc/using/win_installer.png and b/Doc/using/win_installer.png differ diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index c66308e61e89da0..5b715d4614dad8f 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -4,14 +4,14 @@ .. _Microsoft Store app: https://apps.microsoft.com/detail/9NQ7512CXL7T +.. _legacy launcher: https://www.python.org/ftp/python/3.14.0/win32/launcher.msi + .. _using-on-windows: ************************* Using Python on Windows ************************* -.. sectionauthor:: Steve Dower - This document aims to give an overview of Windows-specific behaviour you should know about when using Python on Microsoft Windows. @@ -60,7 +60,7 @@ packages. .. _windows-path-mod: .. _launcher: -Python Install Manager +Python install manager ====================== Installation @@ -103,7 +103,7 @@ Windows Server 2019, please see :ref:`pymanager-advancedinstall` below for more information. -Basic Use +Basic use --------- The recommended command for launching Python is ``python``, which will either @@ -124,9 +124,8 @@ is also an unambiguous ``pymanager`` command. Scripted installs that are intending to use Python install manager should consider using ``pymanager``, due to the lower chance of encountering a conflict with existing installs. The only difference between the two commands is when running without any arguments: -``py`` will install and launch your default interpreter, while ``pymanager`` -will display help (``pymanager exec ...`` provides equivalent behaviour to -``py ...``). +``py`` will launch your default interpreter, while ``pymanager`` will display +help (``pymanager exec ...`` provides equivalent behaviour to ``py ...``). Each of these commands also has a windowed version that avoids creating a console window. These are ``pyw``, ``pythonw`` and ``pymanagerw``. A ``python3`` @@ -185,15 +184,14 @@ that virtual environment. In this scenario, the ``python`` command was likely already overridden and none of these checks occurred. However, this behaviour ensures that the ``py`` command can be used interchangeably. -When you launch either ``python`` or ``py`` but do not have any runtimes -installed, and the requested version is the default, it will be installed -automatically and then launched. Otherwise, the requested version will be -installed if automatic installation is configured (most likely by setting -``PYTHON_MANAGER_AUTOMATIC_INSTALL`` to ``true``), or if the ``py exec`` or -``pymanager exec`` forms of the command were used. +When no runtimes are installed, any launch command will try to install the +requested version and launch it. However, after any version is installed, only +the ``py exec ...`` and ``pymanager exec ...`` commands will install if the +requested version is absent. Other forms of commands will display an error and +direct you to use ``py install`` first. -Command Help +Command help ------------ The ``py help`` command will display the full list of supported commands, along @@ -218,7 +216,7 @@ override multiple settings at once. See :ref:`pymanager-config` below for more information about these files. -Listing Runtimes +Listing runtimes ---------------- .. code:: @@ -259,7 +257,7 @@ For compatibility with the old launcher, the ``--list``, ``--list-paths``, additional options, and will produce legacy formatted output. -Installing Runtimes +Installing runtimes ------------------- .. code:: @@ -287,18 +285,36 @@ work. Passing ``--dry-run`` will generate output and logs, but will not modify any installs. +Passing ``--refresh`` will update all registrations for installed runtimes. This +will recreate Start menu shortcuts, registry keys, and global aliases (such as +``python3.14.exe`` or for any installed scripts). These are automatically +refreshed on installation of any runtime, but may need to be manually refreshed +after installing packages. + In addition to the above options, the ``--target`` option will extract the -runtime to the specified directory instead of doing a normal install. This is -useful for embedding runtimes into larger applications. +runtime to the specified directory instead of doing a normal install. +This is useful for embedding runtimes into larger applications. +Unlike a normal install, ``py`` will not be aware of the extracted runtime, +and no Start menu or other shortcuts will be created. +To launch the runtime, directly execute the main executable (typically +``python.exe``) in the target directory. .. code:: $> py install ... [-t=|--target=] +The ``py exec`` command will install the requested runtime if it is not already +present. This is controlled by the ``automatic_install`` configuration +(:envvar:`PYTHON_MANAGER_AUTOMATIC_INSTALL`), and is enabled by default. +If no runtimes are available at all, all launch commands will do an automatic +install if the configuration setting allows. This is to ensure a good experience +for new users, but should not generally be relied on rather than using the +``py exec`` command or explicit install commands. + .. _pymanager-offline: -Offline Installs +Offline installs ---------------- To perform offline installs of Python, you will need to first create an offline @@ -330,7 +346,7 @@ In this way, Python runtimes can be installed and managed on a machine without access to the internet. -Uninstalling Runtimes +Uninstalling runtimes --------------------- .. code:: @@ -376,10 +392,13 @@ overridden installs may resolve settings differently. A global configuration file may be configured by an administrator, and would be read first. The user configuration file is stored at -:file:`%AppData%\\Python\\pymanager.json` (by default) and is read next, +:file:`%AppData%\\Python\\pymanager.json` +(note that this location is under ``Roaming``, not ``Local``) and is read next, overwriting any settings from earlier files. An additional configuration file may be specified as the ``PYTHON_MANAGER_CONFIG`` environment variable or the ``--config`` command line option (but not both). +These locations may be modified by administrative customization options listed +later. The following settings are those that are considered likely to be modified in normal use. Later sections list those that are intended for administrative @@ -398,7 +417,7 @@ customization. - Description * - ``default_tag`` - - ``PYTHON_MANAGER_DEFAULT`` + - .. envvar:: PYTHON_MANAGER_DEFAULT - The preferred default version to launch or install. By default, this is interpreted as the most recent non-prerelease version from the CPython team. @@ -417,9 +436,11 @@ customization. By default, :file:`%TEMP%`. * - ``automatic_install`` - - ``PYTHON_MANAGER_AUTOMATIC_INSTALL`` - - True to allow automatic installs when specifying a particular runtime - to launch. + - .. envvar:: PYTHON_MANAGER_AUTOMATIC_INSTALL + - True to allow automatic installs when using ``py exec`` to launch (or + ``py`` when no runtimes are installed yet). + Other commands will not automatically install, regardless of this + setting. By default, true. * - ``include_unmanaged`` @@ -434,6 +455,12 @@ customization. Python runtimes, or false to prevent it. By default, true. + * - ``shebang_templates`` + - (none) + - Mapping from shebang line template to alternative command, such as + ``py -V:`` or a substitute string. + See :ref:`pymanager-shebang` for more details. + * - ``log_level`` - ``PYMANAGER_VERBOSE``, ``PYMANAGER_DEBUG`` - Set the default level of output (0-50). @@ -452,11 +479,39 @@ customization. - ``PYTHON_MANAGER_SOURCE_URL`` - Override the index feed to obtain new installs from. + * - ``install.enable_entrypoints`` + - (none) + - True to generate global commands for installed packages (such as + ``pip.exe``). These are defined by the packages themselves. + If set to false, only the Python interpreter has global commands created. + By default, true. You should run ``py install --refresh`` after changing + this setting. + * - ``list.format`` - ``PYTHON_MANAGER_LIST_FORMAT`` - Specify the default format used by the ``py list`` command. By default, ``table``. + * - ``install_dir`` + - (none) + - Specify the root directory that runtimes will be installed into. + If you change this setting, previously installed runtimes will not be + usable unless you move them to the new location. + + * - ``global_dir`` + - (none) + - Specify the directory where global commands (such as ``python3.14.exe`` + and ``pip.exe``) are stored. + This directory should be added to your :envvar:`PATH` to make the + commands available from your terminal. + + * - ``download_dir`` + - (none) + - Specify the directory where downloaded files are stored. + This directory is a temporary cache, and can be cleaned up from time to + time. + + Dotted names should be nested inside JSON objects, for example, ``list.format`` would be specified as ``{"list": {"format": "table"}}``. @@ -519,17 +574,38 @@ which the path to the script and any additional arguments will be appended. This functionality may be disabled by the ``shebang_can_run_anything`` configuration option. +Since version 26.3 of the Python install manager, custom shebang templates may +be added to your configuration file. Add the ``shebang_templates`` object with +one member for each template (the string to match) and the command to use when +the template is matched. Most commands should be ``py -V:`` (or ``pyw``) to +launch one of your installed runtimes. The ``py -3.`` form is also +allowed, as is a plain ``py`` to launch the default. No other arguments are +supported. + +.. code:: json5 + + { + "shebang_templates": { + "/usr/bin/python": "py", + "/usr/bin/my_custom_python": "py -V:MyCustomPython/3" + } + } + +If the substitute command is not ``py`` or ``pyw``, it will be written back into +the shebang and regular handling continues. If launching arbitrary executables +is permitted, then providing a full path will allow you to redirect from Python +to any executable. The template should match either the entire line (ignoring +leading and trailing whitespace), or up to the first space in the shebang line. + + .. note:: The behaviour of shebangs in the Python install manager is subtly different from the previous ``py.exe`` launcher, and the old configuration options no longer apply. If you are specifically reliant on the old behaviour or - configuration, we recommend keeping the legacy launcher. It may be - `downloaded independently `_ - and installed on its own. The legacy launcher's ``py`` command will override - PyManager's one, and you will need to use ``pymanager`` commands for - installing and uninstalling. - + configuration, we recommend installing the `legacy launcher`_. The legacy + launcher's ``py`` command will override PyManager's one by default, and you + will need to use ``pymanager`` commands for installing and uninstalling. .. _Add-AppxPackage: https://learn.microsoft.com/powershell/module/appx/add-appxpackage @@ -541,7 +617,7 @@ configuration option. .. _pymanager-advancedinstall: -Advanced Installation +Advanced installation --------------------- For situations where an MSIX cannot be installed, such as some older @@ -635,7 +711,7 @@ the Store package in this way. .. _pymanager-admin-config: -Administrative Configuration +Administrative configuration ---------------------------- There are a number of options that may be useful for administrators to override @@ -706,6 +782,14 @@ directory containing the configuration file that specified them. (e.g. ``"pep514,start"``). Disabled shortcuts are not reactivated by ``enable_shortcut_kinds``. + * - ``install.hard_link_entrypoints`` + - True to use hard links for global shortcuts to save disk space. If false, + each shortcut executable is copied instead. After changing this setting, + you must run ``py install --refresh --force`` to update existing + commands. + By default, true. Disabling this may be necessary for troubleshooting or + systems that have issues with file links. + * - ``pep514_root`` - Registry location to read and write PEP 514 entries into. By default, :file:`HKEY_CURRENT_USER\\Software\\Python`. @@ -724,19 +808,22 @@ directory containing the configuration file that specified them. - True to suppress visible warnings when a shebang launches an application other than a Python runtime. -.. _install-freethreaded-windows: + * - ``source_settings`` + - A mapping from source URL to settings specific to that index. + When multiple configuration files include this section, URL settings are + added or overwritten, but individual settings are not merged. + These settings are currently only for :ref:`index signatures + `. -Installing Free-threaded Binaries ---------------------------------- -.. versionadded:: 3.13 (Experimental) +.. _install-freethreaded-windows: -.. note:: +Installing free-threaded binaries +--------------------------------- - Everything described in this section is considered experimental, - and should be expected to change in future releases. +.. versionadded:: 3.13 -Pre-built distributions of the experimental free-threaded build are available +Pre-built distributions of the free-threaded build are available by installing tags with the ``t`` suffix. .. code:: @@ -750,6 +837,101 @@ installed, then ``python`` will launch this one. Otherwise, you will need to use ``py -V:3.14t ...`` or, if you have added the global aliases directory to your :envvar:`PATH` environment variable, the ``python3.14t.exe`` commands. + +.. _pymanager-index-signatures: + +Index signatures +---------------- + +.. versionadded:: 26.2 + +Index files may be signed to detect tampering. A signature is a catalog file +at the same URL as the index with ``.cat`` added to the filename. The catalog +file should contain the hash of its matching index file, and should be signed +with a valid Authenticode signature. This allows standard tooling (on Windows) +to generate a signature, and any certificate may be used as long as the client +operating system already trusts its certification authority (root CA). + +Index signatures are only downloaded and checked when the local configuration's +``source_settings`` section includes the index URL and ``requires_signature`` is +true, or the index JSON contains ``requires_signature`` set to true. When the +setting exists in local configuration, even when false, settings in the index +are ignored. + +As well as requiring a valid signature, the ``required_root_subject`` and +``required_publisher_subject`` settings can further restrict acceptable +signatures based on the certificate Subject fields. Any attribute specified in +the configuration must match the attribute in the certificate (additional +attributes in the certificate are ignored). Typical attributes are ``CN=`` for +the common name, ``O=`` for the organizational unit, and ``C=`` for the +publisher's country. + +Finally, the ``required_publisher_eku`` setting allows requiring that a specific +Enhanced Key Usage (EKU) has been assigned to the publisher certificate. For +example, the EKU ``1.3.6.1.5.5.7.3.3`` indicates that the certificate was +intended for code signing (as opposed to server or client authentication). +In combination with a specific root CA, this provides another mechanism to +verify a legitimate signature. + +This is an example ``source_settings`` section from a configuration file. In +this case, the publisher of the feed is uniquely identified by the combination +of the Microsoft Identity Verification root and the EKU assigned by that root. +The signature for this case would be found at +``https://www.python.org/ftp/python/index-windows.json.cat``. + +.. code:: json5 + + { + "source_settings": { + "https://www.python.org/ftp/python/index-windows.json": { + "requires_signature": true, + "required_root_subject": "CN=Microsoft Identity Verification Root Certificate Authority 2020", + "required_publisher_subject": "CN=Python Software Foundation", + "required_publisher_eku": "1.3.6.1.4.1.311.97.608394634.79987812.305991749.578777327" + } + } + } + +The same settings could be specified in the ``index.json`` file instead. In this +case, the root and EKU are omitted, meaning that the signature must be valid and +have a specific common name in the publisher's certificate, but no other checks +are used. + +.. code:: json5 + + { + "requires_signature": true, + "required_publisher_subject": "CN=Python Software Foundation", + "versions": [ + // ... + ] + } + +When settings from inside a feed are used, the user is notified and the settings +are shown in the log file or verbose output. It is recommended to copy these +settings into a local configuration file for feeds that will be used frequently, +so that unauthorised modifications to the feed cannot disable verification. + +It is not possible to override the location of the signature file in the feed or +through a configuration file. Administrators can provide their own +``source_settings`` in a mandatory configuration file (see +:ref:`pymanager-admin-config`). + +If signature validation fails, you will be notified and prompted to continue. +When interactive confirmation is not allowed (for example, because ``--yes`` was +specified), it will always abort. To use a feed with invalid configuration in +this scenario, you must provide a configuration file that disables signature +checking for that feed. + +.. code:: json5 + + "source_settings": { + "https://www.example.com/feed-with-invalid-signature.json": { + "requires_signature": false + } + } + + .. _pymanager-troubleshoot: Troubleshooting @@ -757,7 +939,7 @@ Troubleshooting If your Python install manager does not seem to be working correctly, please work through these tests and fixes to see if it helps. If not, please report an -issue at `our bug tracker `_, +issue at `our bug tracker `_, including any relevant log files (written to your :file:`%TEMP%` directory by default). @@ -786,6 +968,12 @@ default). * - - Check that the ``py`` and ``pymanager`` commands work. + * - + - Ensure your :envvar:`PATH` variable contains the entry for + ``%UserProfile%\AppData\Local\Microsoft\WindowsApps``. + The operating system includes this entry once by default, after other + user paths. If removed, shortcuts will not be found. + * - ``py`` gives me a "command not found" error when I type it in my terminal. - Did you :ref:`install the Python install manager `? @@ -796,6 +984,12 @@ default). The "Python (default windowed)" and "Python install manager" commands may also need refreshing. + * - + - Ensure your :envvar:`PATH` variable contains the entry for + ``%UserProfile%\AppData\Local\Microsoft\WindowsApps``. + The operating system includes this entry once by default, after other + user paths. If removed, shortcuts will not be found. + * - ``py`` gives me a "can't open file" error when I type commands in my terminal. - This usually means you have the legacy launcher installed and @@ -812,7 +1006,7 @@ default). ``python.exe`` alias is set to "Python (default)" * - ``python`` and ``py`` don't launch the runtime I expect - - Check your ``PYTHON_MANAGER_DEFAULT`` environment variable + - Check your :envvar:`PYTHON_MANAGER_DEFAULT` environment variable or ``default_tag`` configuration. The ``py list`` command will show your default based on these settings. @@ -826,7 +1020,7 @@ default). - Prerelease and experimental installs that are not managed by the Python install manager may be chosen ahead of stable releases. Configure your default tag or uninstall the prerelease runtime - and reinstall using ``py install``. + and reinstall it using ``py install``. * - ``pythonw`` or ``pyw`` don't launch the same runtime as ``python`` or ``py`` - Click Start, open "Manage app execution aliases", and check that your @@ -838,13 +1032,48 @@ default). * - - The package may be available but missing the generated executable. - We recommend using the ``python -m pip`` command instead, - or alternatively the ``python -m pip install --force pip`` command - will recreate the executables and show you the path to - add to :envvar:`PATH`. - These scripts are separated for each runtime, and so you may need to - add multiple paths. + We recommend using the ``python -m pip`` command instead. + Running ``py install --refresh`` and ensuring that the global shortcuts + directory is on :envvar:`PATH` (it will be shown in the command output if + it is not) should make commands such as ``pip`` (and other installed + packages) available. + * - I installed a package with ``pip`` but its command is not found. + - Have you activated a virtual environment? + Run the ``.venv\Scripts\activate`` script in your terminal to activate. + + * - + - New packages do not automatically have global shortcuts created by the + Python install manager. Similarly, uninstalled packages do not have their + shortcuts removed. + Run ``py install --refresh`` to update the global shortcuts for newly + installed packages. + + * - Typing ``script-name.py`` in the terminal opens in a new window. + - This is a known limitation of the operating system. Either specify ``py`` + before the script name, create a batch file containing ``@py "%~dpn0.py" %*`` + with the same name as the script, or install the `legacy launcher`_ + and select it as the association for scripts. + + * - Drag-dropping files onto a script doesn't work + - This is a known limitation of the operating system. It is supported with + the `legacy launcher`_, or with the Python install manager when installed + from the MSI. + + * - I have installed the Python install manager multiple times. + - It is possible to install from the Store or WinGet, from the MSIX on + the Python website, and from the MSI, all at once. + They are all compatible and will share configuration and runtimes. + + * - + - See the earlier :ref:`pymanager-advancedinstall` section for ways to + uninstall the install manager other than the typical Installed Apps + (Add and Remove Programs) settings page. + + * - My old ``py.ini`` settings no longer work. + - The new Python install manager no longer supports this configuration file + or its settings, and so it will be ignored. + See :ref:`pymanager-config` for information about configuration settings. .. _windows-embeddable: @@ -862,7 +1091,7 @@ To install an embedded distribution, we recommend using ``py install`` with the .. code:: - $> py install 3.14-embed --target=runtime + $> py install 3.14-embed --target= When extracted, the embedded distribution is (almost) fully isolated from the user's system, including environment variables, system registry settings, and @@ -885,7 +1114,7 @@ versions before providing updates to users. The two recommended use cases for this distribution are described below. -Python Application +Python application ------------------ An application written in Python does not necessarily require users to be aware @@ -989,12 +1218,7 @@ for the 64-bit version, `www.nuget.org/packages/pythonx86 Free-threaded packages ---------------------- -.. versionadded:: 3.13 (Experimental) - -.. note:: - - Everything described in this section is considered experimental, - and should be expected to change in future releases. +.. versionadded:: 3.13 Packages containing free-threaded binaries are named `python-freethreaded `_ @@ -1046,7 +1270,7 @@ please install Python 3.12. .. _max-path: -Removing the MAX_PATH Limitation +Removing the MAX_PATH limitation ================================ Windows historically has limited path lengths to 260 characters. This meant that @@ -1070,7 +1294,7 @@ UTF-8 mode ========== .. versionadded:: 3.7 -.. versionchanged:: next +.. versionchanged:: 3.15 Python UTF-8 mode is now enabled by default (:pep:`686`). @@ -1332,7 +1556,7 @@ installation". In this case: * Shortcuts are available for all users -Removing the MAX_PATH Limitation +Removing the MAX_PATH limitation -------------------------------- Windows historically has limited path lengths to 260 characters. This meant that @@ -1355,7 +1579,7 @@ After changing the above option, no further configuration is required. .. _install-quiet-option: -Installing Without UI +Installing without UI --------------------- All of the options available in the installer UI can also be specified from the @@ -1504,7 +1728,7 @@ example file sets the same options as the previous example: .. _install-layout-option: -Installing Without Downloading +Installing without downloading ------------------------------ As some features of Python are not included in the initial installer download, @@ -1545,15 +1769,10 @@ settings and replace any that have been removed or modified. :ref:`launcher`, which has its own entry in Programs and Features. -Installing Free-threaded Binaries +Installing free-threaded binaries --------------------------------- -.. versionadded:: 3.13 (Experimental) - -.. note:: - - Everything described in this section is considered experimental, - and should be expected to change in future releases. +.. versionadded:: 3.13 To install pre-built binaries with free-threading enabled (see :pep:`703`), you should select "Customize installation". The second page of options includes the @@ -1585,7 +1804,7 @@ builds. Free-threaded binaries are also available :ref:`on nuget.org `. -Python Launcher for Windows (Deprecated) +Python launcher for Windows (deprecated) ======================================== .. deprecated:: 3.14 @@ -1737,7 +1956,7 @@ have the script specify the version which should be used. The key benefit of this is that a single launcher can support multiple Python versions at the same time depending on the contents of the first line. -Shebang Lines +Shebang lines ------------- If the first line of a script file starts with ``#!``, it is known as a @@ -1802,7 +2021,7 @@ program, which performs a :envvar:`PATH` search. If an executable matching the first argument after the ``env`` command cannot be found, but the argument starts with ``python``, it will be handled as described for the other virtual commands. -The environment variable :envvar:`PYLAUNCHER_NO_SEARCH_PATH` may be set +The environment variable :envvar:`!PYLAUNCHER_NO_SEARCH_PATH` may be set (to any value) to skip this search of :envvar:`PATH`. Shebang lines that do not match any of these patterns are looked up in the @@ -1869,7 +2088,7 @@ For example, a shebang line of ``#!python`` has no version qualifier, while ``#!python3`` has a version qualifier which specifies only a major version. If no version qualifiers are found in a command, the environment -variable :envvar:`PY_PYTHON` can be set to specify the default version +variable :envvar:`!PY_PYTHON` can be set to specify the default version qualifier. If it is not set, the default is "3". The variable can specify any value that may be passed on the command line, such as "3", "3.7", "3.7-32" or "3.7-64". (Note that the "-64" option is only @@ -1942,17 +2161,17 @@ For example: Diagnostics ----------- -If an environment variable :envvar:`PYLAUNCHER_DEBUG` is set (to any value), the +If an environment variable :envvar:`!PYLAUNCHER_DEBUG` is set (to any value), the launcher will print diagnostic information to stderr (i.e. to the console). While this information manages to be simultaneously verbose *and* terse, it should allow you to see what versions of Python were located, why a particular version was chosen and the exact command-line used to execute the target Python. It is primarily intended for testing and debugging. -Dry Run +Dry run ------- -If an environment variable :envvar:`PYLAUNCHER_DRYRUN` is set (to any value), +If an environment variable :envvar:`!PYLAUNCHER_DRYRUN` is set (to any value), the launcher will output the command it would have run, but will not actually launch Python. This may be useful for tools that want to use the launcher to detect and then launch Python directly. Note that the command written to @@ -1962,14 +2181,14 @@ the console. Install on demand ----------------- -If an environment variable :envvar:`PYLAUNCHER_ALLOW_INSTALL` is set (to any +If an environment variable :envvar:`!PYLAUNCHER_ALLOW_INSTALL` is set (to any value), and the requested Python version is not installed but is available on the Microsoft Store, the launcher will attempt to install it. This may require user interaction to complete, and you may need to run the command again. -An additional :envvar:`PYLAUNCHER_ALWAYS_INSTALL` variable causes the launcher +An additional :envvar:`!PYLAUNCHER_ALWAYS_INSTALL` variable causes the launcher to always try to install Python, even if it is detected. This is mainly intended -for testing (and should be used with :envvar:`PYLAUNCHER_DRYRUN`). +for testing (and should be used with :envvar:`!PYLAUNCHER_DRYRUN`). Return codes ------------ diff --git a/Doc/whatsnew/2.0.rst b/Doc/whatsnew/2.0.rst index 1a949ec40358072..c157c0337a13427 100644 --- a/Doc/whatsnew/2.0.rst +++ b/Doc/whatsnew/2.0.rst @@ -656,7 +656,8 @@ break. The change which will probably break the most code is tightening up the arguments accepted by some methods. Some methods would take multiple arguments and treat them as a tuple, particularly various list methods such as -:meth:`!append` and :meth:`!insert`. In earlier versions of Python, if ``L`` is +:meth:`~list.append` and :meth:`~list.insert`. +In earlier versions of Python, if ``L`` is a list, ``L.append( 1,2 )`` appends the tuple ``(1,2)`` to the list. In Python 2.0 this causes a :exc:`TypeError` exception to be raised, with the message: 'append requires exactly 1 argument; 2 given'. The fix is to simply add an diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index b7e4e73f4ce4aa0..43ab19037d26273 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -66,7 +66,7 @@ Here's a simple example:: The union and intersection of sets can be computed with the :meth:`~frozenset.union` and :meth:`~frozenset.intersection` methods; an alternative notation uses the bitwise operators ``&`` and ``|``. Mutable sets also have in-place versions of these methods, -:meth:`!union_update` and :meth:`~frozenset.intersection_update`. :: +:meth:`!union_update` and :meth:`~set.intersection_update`. :: >>> S1 = sets.Set([1,2,3]) >>> S2 = sets.Set([4,5,6]) @@ -87,7 +87,7 @@ It's also possible to take the symmetric difference of two sets. This is the set of all elements in the union that aren't in the intersection. Another way of putting it is that the symmetric difference contains all elements that are in exactly one set. Again, there's an alternative notation (``^``), and an -in-place version with the ungainly name :meth:`~frozenset.symmetric_difference_update`. :: +in-place version with the ungainly name :meth:`~set.symmetric_difference_update`. :: >>> S1 = sets.Set([1,2,3,4]) >>> S2 = sets.Set([3,4,5,6]) @@ -1698,8 +1698,8 @@ current local date. Once created, instances of the date/time classes are all immutable. There are a number of methods for producing formatted strings from objects:: - >>> import datetime - >>> now = datetime.datetime.now() + >>> import datetime as dt + >>> now = dt.datetime.now() >>> now.isoformat() '2002-12-30T21:27:03.994956' >>> now.ctime() # Only available on date, datetime @@ -1710,10 +1710,10 @@ number of methods for producing formatted strings from objects:: The :meth:`~datetime.datetime.replace` method allows modifying one or more fields of a :class:`~datetime.date` or :class:`~datetime.datetime` instance, returning a new instance:: - >>> d = datetime.datetime.now() + >>> d = dt.datetime.now() >>> d datetime.datetime(2002, 12, 30, 22, 15, 38, 827738) - >>> d.replace(year=2001, hour = 12) + >>> d.replace(year=2001, hour=12) datetime.datetime(2001, 12, 30, 12, 15, 38, 827738) >>> diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 3430ac8668e280c..4b92ba82c991e44 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -745,7 +745,7 @@ The contextlib module The new :mod:`contextlib` module provides some functions and a decorator that are useful for writing objects for use with the ':keyword:`with`' statement. -The decorator is called :func:`contextmanager`, and lets you write a single +The decorator is called :deco:`~contextlib.contextmanager`, and lets you write a single generator function instead of defining a new class. The generator should yield exactly one value. The code up to the :keyword:`yield` will be executed as the :meth:`~object.__enter__` method, and the value yielded will be the method's return @@ -1277,12 +1277,12 @@ complete list of changes, or look through the SVN logs for all the details. with the new ':keyword:`with`' statement. See section :ref:`contextlibmod` for more about this module. -* New module: The :mod:`cProfile` module is a C implementation of the existing - :mod:`profile` module that has much lower overhead. The module's interface is - the same as :mod:`profile`: you run ``cProfile.run('main()')`` to profile a +* New module: The :mod:`!cProfile` module is a C implementation of the existing + :mod:`!profile` module that has much lower overhead. The module's interface is + the same as :mod:`!profile`: you run ``cProfile.run('main()')`` to profile a function, can save profile data to a file, etc. It's not yet known if the Hotshot profiler, which is also written in C but doesn't match the - :mod:`profile` module's interface, will continue to be maintained in future + :mod:`!profile` module's interface, will continue to be maintained in future versions of Python. (Contributed by Armin Rigo.) Also, the :mod:`pstats` module for analyzing the data measured by the profiler @@ -1313,10 +1313,10 @@ complete list of changes, or look through the SVN logs for all the details. by Josh Spoerri. It uses the same format characters as :func:`time.strptime` and :func:`time.strftime`:: - from datetime import datetime + import datetime as dt - ts = datetime.strptime('10:13:15 2006-03-07', - '%H:%M:%S %Y-%m-%d') + ts = dt.datetime.strptime('10:13:15 2006-03-07', + '%H:%M:%S %Y-%m-%d') * The :meth:`SequenceMatcher.get_matching_blocks` method in the :mod:`difflib` module now guarantees to return a minimal list of blocks describing matching @@ -2169,9 +2169,9 @@ Changes to Python's build process and to the C API include: * Two new macros can be used to indicate C functions that are local to the current file so that a faster calling convention can be used. - ``Py_LOCAL(type)`` declares the function as returning a value of the + :c:macro:`Py_LOCAL` declares the function as returning a value of the specified *type* and uses a fast-calling qualifier. - ``Py_LOCAL_INLINE(type)`` does the same thing and also requests the + :c:macro:`Py_LOCAL_INLINE` does the same thing and also requests the function be inlined. If macro :c:macro:`!PY_LOCAL_AGGRESSIVE` is defined before :file:`python.h` is included, a set of more aggressive optimizations are enabled for the module; you should benchmark the results to find out if these diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 0803eba99e6d172..e1f72b6c4a6c92c 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -56,7 +56,7 @@ Python 2.6 incorporates new features and syntax from 3.0 while remaining compatible with existing code by not removing older features or syntax. When it's not possible to do that, Python 2.6 tries to do what it can, adding compatibility functions in a -:mod:`future_builtins` module and a :option:`!-3` switch to warn about +:mod:`!future_builtins` module and a :option:`!-3` switch to warn about usages that will become unsupported in 3.0. Some significant new packages have been added to the standard library, @@ -109,7 +109,7 @@ are: Python 3.0 adds several new built-in functions and changes the semantics of some existing builtins. Functions that are new in 3.0 such as :func:`bin` have simply been added to Python 2.6, but existing -builtins haven't been changed; instead, the :mod:`future_builtins` +builtins haven't been changed; instead, the :mod:`!future_builtins` module has versions with the new 3.0 semantics. Code written to be compatible with 3.0 can do ``from future_builtins import hex, map`` as necessary. @@ -118,7 +118,7 @@ A new command-line switch, :option:`!-3`, enables warnings about features that will be removed in Python 3.0. You can run code with this switch to see how much work will be necessary to port code to 3.0. The value of this switch is available -to Python code as the boolean variable :data:`sys.py3kwarning`, +to Python code as the boolean variable :data:`!sys.py3kwarning`, and to C extension code as :c:data:`!Py_Py3kWarningFlag`. .. seealso:: @@ -307,9 +307,9 @@ The :mod:`threading` module's locks and condition variables also support the The lock is acquired before the block is executed and always released once the block is complete. -The :func:`localcontext` function in the :mod:`decimal` module makes it easy -to save and restore the current decimal context, which encapsulates the desired -precision and rounding characteristics for computations:: +The :func:`~decimal.localcontext` function in the :mod:`decimal` module makes +it easy to save and restore the current decimal context, which encapsulates +the desired precision and rounding characteristics for computations:: from decimal import Decimal, Context, localcontext @@ -337,12 +337,12 @@ underlying implementation and should keep reading. A high-level explanation of the context management protocol is: * The expression is evaluated and should result in an object called a "context - manager". The context manager must have :meth:`~object.__enter__` and :meth:`~object.__exit__` - methods. + manager". The context manager must have :meth:`~object.__enter__` and + :meth:`~object.__exit__` methods. -* The context manager's :meth:`~object.__enter__` method is called. The value returned - is assigned to *VAR*. If no ``as VAR`` clause is present, the value is simply - discarded. +* The context manager's :meth:`~object.__enter__` method is called. The value + returned is assigned to *VAR*. If no ``as VAR`` clause is present, the + value is simply discarded. * The code in *BLOCK* is executed. @@ -378,7 +378,7 @@ be to let the user write code like this:: The transaction should be committed if the code in the block runs flawlessly or rolled back if there's an exception. Here's the basic interface for -:class:`DatabaseConnection` that I'll assume:: +:class:`!DatabaseConnection` that I'll assume:: class DatabaseConnection: # Database interface @@ -431,14 +431,15 @@ The contextlib module The :mod:`contextlib` module provides some functions and a decorator that are useful when writing objects for use with the ':keyword:`with`' statement. -The decorator is called :func:`contextmanager`, and lets you write a single -generator function instead of defining a new class. The generator should yield -exactly one value. The code up to the :keyword:`yield` will be executed as the -:meth:`~object.__enter__` method, and the value yielded will be the method's return -value that will get bound to the variable in the ':keyword:`with`' statement's -:keyword:`!as` clause, if any. The code after the :keyword:`!yield` will be -executed in the :meth:`~object.__exit__` method. Any exception raised in the block will -be raised by the :keyword:`!yield` statement. +The decorator is called :deco:`~contextlib.contextmanager`, and lets you write +a single generator function instead of defining a new class. The generator +should yield exactly one value. The code up to the :keyword:`yield` will be +executed as the :meth:`~object.__enter__` method, and the value yielded will +be the method's return value that will get bound to the variable in the +':keyword:`with`' statement's :keyword:`!as` clause, if any. The code after +the :keyword:`!yield` will be executed in the :meth:`~object.__exit__` method. +Any exception raised in the block will be raised by the :keyword:`!yield` +statement. Using this decorator, our database example from the previous section could be written as:: @@ -469,7 +470,7 @@ statement both starts a database transaction and acquires a thread lock:: with nested (db_transaction(db), lock) as (cursor, locked): ... -Finally, the :func:`closing` function returns its argument so that it can be +Finally, the :func:`~contextlib.closing` function returns its argument so that it can be bound to a variable, and calls the argument's ``.close()`` method at the end of the block. :: @@ -538,7 +539,7 @@ If you don't like the default directory, it can be overridden by an environment variable. :envvar:`PYTHONUSERBASE` sets the root directory used for all Python versions supporting this feature. On Windows, the directory for application-specific data can be changed by -setting the :envvar:`APPDATA` environment variable. You can also +setting the :envvar:`!APPDATA` environment variable. You can also modify the :file:`site.py` file for your Python installation. The feature can be disabled entirely by running Python with the @@ -568,11 +569,12 @@ The :mod:`multiprocessing` module started out as an exact emulation of the :mod:`threading` module using processes instead of threads. That goal was discarded along the path to Python 2.6, but the general approach of the module is still similar. The fundamental class -is the :class:`Process`, which is passed a callable object and -a collection of arguments. The :meth:`start` method +is the :class:`~multiprocessing.Process`, which is passed a callable object and +a collection of arguments. The :meth:`~multiprocessing.Process.start` method sets the callable running in a subprocess, after which you can call -the :meth:`is_alive` method to check whether the subprocess is still running -and the :meth:`join` method to wait for the process to exit. +the :meth:`~multiprocessing.Process.is_alive` method to check whether the +subprocess is still running and the :meth:`~multiprocessing.Process.join` +method to wait for the process to exit. Here's a simple example where the subprocess will calculate a factorial. The function doing the calculation is written strangely so @@ -619,13 +621,16 @@ the object to communicate. (If the parent were to change the value of the global variable, the child's value would be unaffected, and vice versa.) -Two other classes, :class:`Pool` and :class:`Manager`, provide -higher-level interfaces. :class:`Pool` will create a fixed number of -worker processes, and requests can then be distributed to the workers -by calling :meth:`apply` or :meth:`apply_async` to add a single request, -and :meth:`map` or :meth:`map_async` to add a number of -requests. The following code uses a :class:`Pool` to spread requests -across 5 worker processes and retrieve a list of results:: +Two other classes, :class:`~multiprocessing.pool.Pool` and +:class:`~multiprocessing.Manager`, provide higher-level interfaces. +:class:`~multiprocessing.pool.Pool` will create a fixed number of worker +processes, and requests can then be distributed to the workers by calling +:meth:`~multiprocessing.pool.Pool.apply` or +:meth:`~multiprocessing.pool.Pool.apply_async` to add a single request, and +:meth:`~multiprocessing.pool.Pool.map` or +:meth:`~multiprocessing.pool.Pool.map_async` to add a number of +requests. The following code uses a :class:`~multiprocessing.pool.Pool` to +spread requests across 5 worker processes and retrieve a list of results:: from multiprocessing import Pool @@ -646,15 +651,18 @@ This produces the following output:: 33452526613163807108170062053440751665152000000000 ... -The other high-level interface, the :class:`Manager` class, creates a -separate server process that can hold master copies of Python data +The other high-level interface, the :class:`~multiprocessing.Manager` class, +creates a separate server process that can hold master copies of Python data structures. Other processes can then access and modify these data structures using proxy objects. The following example creates a shared dictionary by calling the :meth:`dict` method; the worker processes then insert values into the dictionary. (Locking is not done for you automatically, which doesn't matter in this example. -:class:`Manager`'s methods also include :meth:`Lock`, :meth:`RLock`, -and :meth:`Semaphore` to create shared locks.) +:class:`~multiprocessing.Manager`'s methods also include +:meth:`~multiprocessing.managers.SyncManager.Lock`, +:meth:`~multiprocessing.managers.SyncManager.RLock`, +and :meth:`~multiprocessing.managers.SyncManager.Semaphore` to create +shared locks.) :: @@ -824,7 +832,7 @@ documentation for a :ref:`complete list `; here's a sample: format, followed by a percent sign. ===== ======================================================================== -Classes and types can define a :meth:`__format__` method to control how they're +Classes and types can define a :meth:`~object.__format__` method to control how they're formatted. It receives a single argument, the format specifier:: def __format__(self, format_spec): @@ -834,7 +842,7 @@ formatted. It receives a single argument, the format specifier:: return str(self) There's also a :func:`format` builtin that will format a single -value. It calls the type's :meth:`__format__` method with the +value. It calls the type's :meth:`~object.__format__` method with the provided specifier:: >>> format(75.6564, '.2f') @@ -997,9 +1005,10 @@ sequence of bytes:: u'\u31ef \u3244' Byte arrays support most of the methods of string types, such as -:meth:`startswith`/:meth:`endswith`, :meth:`find`/:meth:`rfind`, -and some of the methods of lists, such as :meth:`append`, -:meth:`pop`, and :meth:`reverse`. +:meth:`~bytearray.startswith`/:meth:`~bytearray.endswith`, +:meth:`~bytearray.find`/:meth:`~bytearray.rfind`, +and some of the methods of lists, such as :meth:`~bytearray.append`, +:meth:`~bytearray.pop`, and :meth:`~bytearray.reverse`. :: @@ -1028,56 +1037,58 @@ PEP 3116: New I/O Library Python's built-in file objects support a number of methods, but file-like objects don't necessarily support all of them. Objects that -imitate files usually support :meth:`read` and :meth:`write`, but they -may not support :meth:`readline`, for example. Python 3.0 introduces -a layered I/O library in the :mod:`io` module that separates buffering -and text-handling features from the fundamental read and write -operations. +imitate files usually support :meth:`!read` and +:meth:`!write`, but they may not support :meth:`!readline`, +for example. Python 3.0 introduces a layered I/O library in the :mod:`io` +module that separates buffering and text-handling features from the +fundamental read and write operations. There are three levels of abstract base classes provided by the :mod:`io` module: -* :class:`RawIOBase` defines raw I/O operations: :meth:`read`, - :meth:`readinto`, - :meth:`write`, :meth:`seek`, :meth:`tell`, :meth:`truncate`, - and :meth:`close`. +* :class:`~io.RawIOBase` defines raw I/O operations: :meth:`~io.RawIOBase.read`, + :meth:`~io.RawIOBase.readinto`, :meth:`~io.RawIOBase.write`, + :meth:`~io.IOBase.seek`, :meth:`~io.IOBase.tell`, :meth:`~io.IOBase.truncate`, + and :meth:`~io.IOBase.close`. Most of the methods of this class will often map to a single system call. - There are also :meth:`readable`, :meth:`writable`, and :meth:`seekable` - methods for determining what operations a given object will allow. + There are also :meth:`~io.IOBase.readable`, :meth:`~io.IOBase.writable`, + and :meth:`~io.IOBase.seekable` methods for determining what operations a + given object will allow. Python 3.0 has concrete implementations of this class for files and sockets, but Python 2.6 hasn't restructured its file and socket objects in this way. -* :class:`BufferedIOBase` is an abstract base class that +* :class:`~io.BufferedIOBase` is an abstract base class that buffers data in memory to reduce the number of system calls used, making I/O processing more efficient. - It supports all of the methods of :class:`RawIOBase`, - and adds a :attr:`raw` attribute holding the underlying raw object. + It supports all of the methods of :class:`~io.RawIOBase`, + and adds a :attr:`~io.BufferedIOBase.raw` attribute holding the underlying + raw object. There are five concrete classes implementing this ABC. - :class:`BufferedWriter` and :class:`BufferedReader` are for objects - that support write-only or read-only usage that have a :meth:`seek` - method for random access. :class:`BufferedRandom` objects support + :class:`~io.BufferedWriter` and :class:`~io.BufferedReader` are for objects + that support write-only or read-only usage that have a :meth:`~io.IOBase.seek` + method for random access. :class:`~io.BufferedRandom` objects support read and write access upon the same underlying stream, and - :class:`BufferedRWPair` is for objects such as TTYs that have both + :class:`~io.BufferedRWPair` is for objects such as TTYs that have both read and write operations acting upon unconnected streams of data. - The :class:`BytesIO` class supports reading, writing, and seeking + The :class:`~io.BytesIO` class supports reading, writing, and seeking over an in-memory buffer. .. index:: single: universal newlines; What's new -* :class:`TextIOBase`: Provides functions for reading and writing +* :class:`~io.TextIOBase`: Provides functions for reading and writing strings (remember, strings will be Unicode in Python 3.0), - and supporting :term:`universal newlines`. :class:`TextIOBase` defines + and supporting :term:`universal newlines`. :class:`~io.TextIOBase` defines the :meth:`readline` method and supports iteration upon objects. - There are two concrete implementations. :class:`TextIOWrapper` + There are two concrete implementations. :class:`~io.TextIOWrapper` wraps a buffered I/O object, supporting all of the methods for - text I/O and adding a :attr:`buffer` attribute for access - to the underlying object. :class:`StringIO` simply buffers + text I/O and adding a :attr:`~io.TextIOBase.buffer` attribute for access + to the underlying object. :class:`~io.StringIO` simply buffers everything in memory without ever writing anything to disk. (In Python 2.6, :class:`io.StringIO` is implemented in @@ -1161,7 +1172,7 @@ Some object-oriented languages such as Java support interfaces, declaring that a class has a given set of methods or supports a given access protocol. Abstract Base Classes (or ABCs) are an equivalent feature for Python. The ABC support consists of an :mod:`abc` module -containing a metaclass called :class:`ABCMeta`, special handling of +containing a metaclass called :class:`~abc.ABCMeta`, special handling of this metaclass by the :func:`isinstance` and :func:`issubclass` builtins, and a collection of basic ABCs that the Python developers think will be widely useful. Future versions of Python will probably @@ -1171,17 +1182,17 @@ Let's say you have a particular class and wish to know whether it supports dictionary-style access. The phrase "dictionary-style" is vague, however. It probably means that accessing items with ``obj[1]`` works. Does it imply that setting items with ``obj[2] = value`` works? -Or that the object will have :meth:`keys`, :meth:`values`, and :meth:`items` -methods? What about the iterative variants such as :meth:`iterkeys`? :meth:`copy` -and :meth:`update`? Iterating over the object with :func:`iter`? +Or that the object will have :meth:`!keys`, :meth:`!values`, and :meth:`!items` +methods? What about the iterative variants such as :meth:`!iterkeys`? +:meth:`!copy`and :meth:`!update`? Iterating over the object with :func:`!iter`? The Python 2.6 :mod:`collections` module includes a number of different ABCs that represent these distinctions. :class:`Iterable` -indicates that a class defines :meth:`__iter__`, and -:class:`Container` means the class defines a :meth:`__contains__` +indicates that a class defines :meth:`~object.__iter__`, and +:class:`Container` means the class defines a :meth:`~object.__contains__` method and therefore supports ``x in y`` expressions. The basic dictionary interface of getting items, setting items, and -:meth:`keys`, :meth:`values`, and :meth:`items`, is defined by the +:meth:`!keys`, :meth:`!values`, and :meth:`!items`, is defined by the :class:`MutableMapping` ABC. You can derive your own classes from a particular ABC @@ -1195,7 +1206,7 @@ to indicate they support that ABC's interface:: Alternatively, you could write the class without deriving from the desired ABC and instead register the class by -calling the ABC's :meth:`register` method:: +calling the ABC's :meth:`~abc.ABCMeta.register` method:: import collections @@ -1205,10 +1216,10 @@ calling the ABC's :meth:`register` method:: collections.MutableMapping.register(Storage) For classes that you write, deriving from the ABC is probably clearer. -The :meth:`register` method is useful when you've written a new +The :meth:`~abc.ABCMeta.register` method is useful when you've written a new ABC that can describe an existing type or class, or if you want to declare that some third-party class implements an ABC. -For example, if you defined a :class:`PrintableType` ABC, +For example, if you defined a :class:`!PrintableType` ABC, it's legal to do:: # Register Python's types @@ -1255,16 +1266,16 @@ metaclass in a class definition:: ... -In the :class:`Drawable` ABC above, the :meth:`draw_doubled` method +In the :class:`!Drawable` ABC above, the :meth:`!draw_doubled` method renders the object at twice its size and can be implemented in terms -of other methods described in :class:`Drawable`. Classes implementing +of other methods described in :class:`!Drawable`. Classes implementing this ABC therefore don't need to provide their own implementation -of :meth:`draw_doubled`, though they can do so. An implementation -of :meth:`draw` is necessary, though; the ABC can't provide +of :meth:`!draw_doubled`, though they can do so. An implementation +of :meth:`!draw` is necessary, though; the ABC can't provide a useful generic implementation. -You can apply the ``@abstractmethod`` decorator to methods such as -:meth:`draw` that must be implemented; Python will then raise an +You can apply the :deco:`~abc.abstractmethod` decorator to methods such as +:meth:`!draw` that must be implemented; Python will then raise an exception for classes that don't define the method. Note that the exception is only raised when you actually try to create an instance of a subclass lacking the method:: @@ -1288,7 +1299,7 @@ Abstract data attributes can be declared using the def readonly(self): return self._x -Subclasses must then define a :meth:`readonly` property. +Subclasses must then define a ``readonly`` property. .. seealso:: @@ -1528,8 +1539,8 @@ Some smaller changes made to the core Python language are: the :exc:`StopIteration` exception will be raised. (Backported in :issue:`2719`.) -* Tuples now have :meth:`index` and :meth:`count` methods matching the - list type's :meth:`index` and :meth:`count` methods:: +* Tuples now have :meth:`~tuple.index` and :meth:`~tuple.count` methods + matching the list type's :meth:`~list.index` and :meth:`~list.count` methods:: >>> t = (0,1,2,3,4,0,1,2) >>> t.index(3) @@ -1546,8 +1557,9 @@ Some smaller changes made to the core Python language are: .. Revision 57619 -* Properties now have three attributes, :attr:`getter`, :attr:`setter` - and :attr:`deleter`, that are decorators providing useful shortcuts +* Properties now have three decorators, :deco:`~property.getter`, + :deco:`~property.setter` and :deco:`~property.deleter`, that are + decorators providing useful shortcuts for adding a getter, setter or deleter function to an existing property. You would use them like this:: @@ -2738,13 +2750,13 @@ numbers. .. ====================================================================== -The :mod:`future_builtins` module +The :mod:`!future_builtins` module -------------------------------------- Python 3.0 makes many changes to the repertoire of built-in functions, and most of the changes can't be introduced in the Python 2.x series because they would break compatibility. -The :mod:`future_builtins` module provides versions +The :mod:`!future_builtins` module provides versions of these built-in functions that can be imported when writing 3.0-compatible code. @@ -2811,10 +2823,10 @@ Using the module is simple:: import sys import plistlib - import datetime + import datetime as dt # Create data structure - data_struct = dict(lastAccessed=datetime.datetime.now(), + data_struct = dict(lastAccessed=dt.datetime.now(), version=1, categories=('Personal','Shared','Private')) diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index caed3192be871de..de90c37a38d38ff 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -858,8 +858,8 @@ Some smaller changes made to the core Python language are: .. XXX bytearray doesn't seem to be documented -* When using :class:`@classmethod ` and - :class:`@staticmethod ` to wrap +* When using :deco:`classmethod` and + :deco:`staticmethod` to wrap methods as class or static methods, the wrapper object now exposes the wrapped function as their :attr:`~method.__func__` attribute. @@ -1267,7 +1267,7 @@ changes, or look through the Subversion logs for all the details. uploads thanks to an added *rest* parameter (patch by Pablo Mouzo; :issue:`6845`.) -* New class decorator: :func:`~functools.total_ordering` in the :mod:`functools` +* New class decorator: :deco:`~functools.total_ordering` in the :mod:`functools` module takes a class that defines an :meth:`~object.__eq__` method and one of :meth:`~object.__lt__`, :meth:`~object.__le__`, :meth:`~object.__gt__`, or :meth:`~object.__ge__`, and generates the missing comparison methods. Since the @@ -1541,7 +1541,7 @@ changes, or look through the Subversion logs for all the details. buffer API, which fixed a test suite failure (fix by Antoine Pitrou; :issue:`7133`) and automatically set OpenSSL's :c:macro:`!SSL_MODE_AUTO_RETRY`, which will prevent an error - code being returned from :meth:`recv` operations that trigger an SSL + code being returned from :meth:`!recv` operations that trigger an SSL renegotiation (fix by Antoine Pitrou; :issue:`8222`). The :func:`~ssl.SSLContext.wrap_socket` constructor function now takes a @@ -2031,7 +2031,7 @@ version 1.3. Some of the new features are: * ElementTree's code for converting trees to a string has been significantly reworked, making it roughly twice as fast in many cases. The :meth:`ElementTree.write() ` - and :meth:`Element.write` methods now have a *method* parameter that can be + and :meth:`!Element.write` methods now have a *method* parameter that can be "xml" (the default), "html", or "text". HTML mode will output empty elements as ```` instead of ````, and text mode will skip over elements and only output the text chunks. If @@ -2044,7 +2044,7 @@ version 1.3. Some of the new features are: Namespace handling has also been improved. All ``xmlns:`` declarations are now output on the root element, not scattered throughout the resulting XML. You can set the default namespace for a tree - by setting the :attr:`default_namespace` attribute and can + by setting the :attr:`!default_namespace` attribute and can register new prefixes with :meth:`~xml.etree.ElementTree.register_namespace`. In XML mode, you can use the true/false *xml_declaration* parameter to suppress the XML declaration. @@ -2196,8 +2196,6 @@ Changes to Python's build process and to the C API include: locale-independent way. (Added by Eric Smith; :issue:`5793`.) - .. XXX these macros don't seem to be described in the c-api docs. - * Removed function: :c:func:`!PyEval_CallObject` is now only available as a macro. A function version was being kept around to preserve ABI linking compatibility, but that was in 1997; it can certainly be @@ -2234,7 +2232,7 @@ Changes to Python's build process and to the C API include: * When using the :c:type:`PyMemberDef` structure to define attributes of a type, Python will no longer let you try to delete or set a - :c:macro:`T_STRING_INPLACE` attribute. + :c:macro:`!T_STRING_INPLACE` attribute. .. rev 79644 @@ -2259,12 +2257,12 @@ Changes to Python's build process and to the C API include: :issue:`6491`.) * The :program:`configure` script now checks for floating-point rounding bugs - on certain 32-bit Intel chips and defines a :c:macro:`X87_DOUBLE_ROUNDING` + on certain 32-bit Intel chips and defines a :c:macro:`!X87_DOUBLE_ROUNDING` preprocessor definition. No code currently uses this definition, but it's available if anyone wishes to use it. (Added by Mark Dickinson; :issue:`2937`.) - :program:`configure` also now sets a :envvar:`LDCXXSHARED` Makefile + :program:`configure` also now sets a :envvar:`!LDCXXSHARED` Makefile variable for supporting C++ linking. (Contributed by Arfrever Frehtes Taifersar Arahesis; :issue:`1222585`.) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 1067601c652300c..390b82d5a1fdc1b 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -402,7 +402,7 @@ Tracing events, with the correct line number, are generated for all lines of cod The :attr:`~frame.f_lineno` attribute of frame objects will always contain the expected line number. -The :attr:`~codeobject.co_lnotab` attribute of +The :attr:`!codeobject.co_lnotab` attribute of :ref:`code objects ` is deprecated and will be removed in 3.12. Code that needs to convert from offset to line number should use the new @@ -847,8 +847,8 @@ Other Language Changes respectively. (Contributed by Joshua Bronson, Daniel Pope, and Justin Wang in :issue:`31861`.) -* Static methods (:func:`@staticmethod `) and class methods - (:func:`@classmethod `) now inherit the method attributes +* Static methods (:deco:`staticmethod`) and class methods + (:deco:`classmethod`) now inherit the method attributes (``__module__``, ``__name__``, ``__qualname__``, ``__doc__``, ``__annotations__``) and have a new ``__wrapped__`` attribute. Moreover, static methods are now callable as regular functions. @@ -901,7 +901,7 @@ Improved Modules asyncio ------- -Add missing :meth:`~asyncio.events.AbstractEventLoop.connect_accepted_socket` +Add missing :meth:`~asyncio.loop.connect_accepted_socket` method. (Contributed by Alex Grönholm in :issue:`41332`.) @@ -933,7 +933,7 @@ Base32 Encoding with Extended Hex Alphabet. bdb --- -Add :meth:`~bdb.Breakpoint.clearBreakpoints` to reset all set breakpoints. +Add :meth:`!clearBreakpoints` to reset all set breakpoints. (Contributed by Irit Katriel in :issue:`24160`.) bisect @@ -996,7 +996,7 @@ dataclasses __slots__ ~~~~~~~~~ -Added ``slots`` parameter in :func:`dataclasses.dataclass` decorator. +Added ``slots`` parameter in :deco:`dataclasses.dataclass` decorator. (Contributed by Yurii Karabas in :issue:`42269`) Keyword-only fields @@ -1398,7 +1398,7 @@ A new verify flag :const:`~ssl.VERIFY_X509_PARTIAL_CHAIN` has been added. sqlite3 ------- -Add audit events for :func:`~sqlite3.connect/handle`, +Add audit events for :func:`~sqlite3.connect`, :meth:`~sqlite3.Connection.enable_load_extension`, and :meth:`~sqlite3.Connection.load_extension`. (Contributed by Erlend E. Aasland in :issue:`43762`.) @@ -1446,8 +1446,8 @@ as a positional-only argument. types ----- -Reintroduce the :data:`types.EllipsisType`, :data:`types.NoneType` -and :data:`types.NotImplementedType` classes, providing a new set +Reintroduce the :class:`types.EllipsisType`, :class:`types.NoneType` +and :class:`types.NotImplementedType` classes, providing a new set of types readily interpretable by type checkers. (Contributed by Bas van Beek in :issue:`41810`.) @@ -1485,9 +1485,9 @@ is a :class:`typing.TypedDict`. Subclasses of ``typing.Protocol`` which only have data variables declared will now raise a ``TypeError`` when checked with ``isinstance`` unless they -are decorated with :func:`~typing.runtime_checkable`. Previously, these checks +are decorated with :deco:`~typing.runtime_checkable`. Previously, these checks passed silently. Users should decorate their -subclasses with the :func:`!runtime_checkable` decorator +subclasses with the :deco:`!runtime_checkable` decorator if they want runtime protocols. (Contributed by Yurii Karabas in :issue:`38908`.) @@ -1622,7 +1622,7 @@ Deprecated compatibility. Specifically, :meth:`!find_loader`/:meth:`!find_module` (superseded by :meth:`~importlib.abc.MetaPathFinder.find_spec`), - :meth:`~importlib.abc.Loader.load_module` + ``importlib.abc.Loader.load_module`` (superseded by :meth:`~importlib.abc.Loader.exec_module`), :meth:`!module_repr` (which the import system takes care of for you), the ``__package__`` attribute @@ -1652,7 +1652,7 @@ Deprecated preference for :meth:`~zipimport.zipimporter.exec_module`. (Contributed by Brett Cannon in :issue:`26131`.) -* The use of :meth:`~importlib.abc.Loader.load_module` by the import +* The use of ``importlib.abc.Loader.load_module`` by the import system now triggers an :exc:`ImportWarning` as :meth:`~importlib.abc.Loader.exec_module` is preferred. (Contributed by Brett Cannon in :issue:`26131`.) @@ -1928,7 +1928,7 @@ Changes in the Python API (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley in :issue:`42392`.) -* The :data:`types.FunctionType` constructor now inherits the current builtins +* The :class:`types.FunctionType` constructor now inherits the current builtins if the *globals* dictionary has no ``"__builtins__"`` key, rather than using ``{"None": None}`` as builtins: same behavior as :func:`eval` and :func:`exec` functions. Defining a function with ``def function(...): ...`` diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index abf9677fd9cac50..7b3475882063635 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -388,7 +388,7 @@ Kumar Srinivasan and Graham Bleaney.) PEP 681: Data class transforms ------------------------------ -:data:`~typing.dataclass_transform` may be used to +:deco:`~typing.dataclass_transform` may be used to decorate a class, metaclass, or a function that is itself a decorator. The presence of ``@dataclass_transform()`` tells a static type checker that the decorated object performs runtime "magic" that transforms a class, @@ -686,8 +686,8 @@ enum * Added the :func:`~enum.member` and :func:`~enum.nonmember` decorators, to ensure the decorated object is/is not converted to an enum member. -* Added the :func:`~enum.property` decorator, - which works like :func:`property` except for enums. +* Added the :deco:`~enum.property` decorator, + which works like :deco:`property` except for enums. Use this instead of :func:`types.DynamicClassAttribute`. * Added the :func:`~enum.global_enum` enum decorator, @@ -740,7 +740,7 @@ fractions functools --------- -* :func:`functools.singledispatch` now supports :class:`types.UnionType` +* :deco:`functools.singledispatch` now supports :class:`types.UnionType` and :class:`typing.Union` as annotations to the dispatch argument.:: >>> from functools import singledispatch @@ -1164,7 +1164,7 @@ For major changes, see :ref:`new-feat-related-type-hints-311`. type checker errors related to highly dynamic class, such as mocks. (Contributed by Shantanu Jain in :gh:`91154`.) -* The :func:`typing.final` decorator now sets the ``__final__`` attributed on +* The :deco:`typing.final` decorator now sets the ``__final__`` attributed on the decorated object. (Contributed by Jelle Zijlstra in :gh:`90500`.) @@ -1197,7 +1197,7 @@ For major changes, see :ref:`new-feat-related-type-hints-311`. :data:`~typing.ClassVar` annotations. (Contributed by Gregory Beauregard in :gh:`90711`.) -* :func:`typing.no_type_check` no longer modifies external classes and functions. +* :deco:`typing.no_type_check` no longer modifies external classes and functions. It also now correctly marks classmethods as not to be type checked. (Contributed by Nikita Sobolev in :gh:`90729`.) @@ -1292,7 +1292,7 @@ This section covers specific optimizations independent of the (Contributed by Stefan Behnel in :gh:`68264`.) * Resizing lists is streamlined for the common case, - speeding up :meth:`!list.append` by ≈15% + speeding up :meth:`list.append` by ≈15% and simple :term:`list comprehension`\s by up to 20-30% (Contributed by Dennis Sweeney in :gh:`91165`.) @@ -2673,7 +2673,7 @@ Removed (Contributed by Victor Stinner in :issue:`45474`.) -* Exclude :c:func:`PyWeakref_GET_OBJECT` from the limited C API. It never +* Exclude :c:func:`!PyWeakref_GET_OBJECT` from the limited C API. It never worked since the :c:type:`!PyWeakReference` structure is opaque in the limited C API. (Contributed by Victor Stinner in :issue:`35134`.) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 7cfdc287b7fad72..9e48b3ab7393c87 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -146,7 +146,7 @@ New typing features: * :ref:`PEP 692 `, using :class:`~typing.TypedDict` to annotate :term:`**kwargs ` -* :ref:`PEP 698 `, :func:`typing.override` decorator +* :ref:`PEP 698 `, :deco:`typing.override` decorator Important deprecations, removals or restrictions: @@ -511,7 +511,7 @@ See :pep:`692` for more details. PEP 698: Override Decorator for Static Typing --------------------------------------------- -A new decorator :func:`typing.override` has been added to the :mod:`typing` +A new decorator :deco:`typing.override` has been added to the :mod:`typing` module. It indicates to type checkers that the method is intended to override a method in a superclass. This allows type checkers to catch mistakes where a method that is intended to override something in a base class @@ -1021,7 +1021,7 @@ typing ``__orig_bases__`` attribute. (Contributed by Adrian Garcia Badaracco in :gh:`103699`.) -* Add ``frozen_default`` parameter to :func:`typing.dataclass_transform`. +* Add ``frozen_default`` parameter to :deco:`typing.dataclass_transform`. (Contributed by Erik De Bonte in :gh:`99957`.) unicodedata @@ -1191,9 +1191,23 @@ Deprecated replaced by :data:`calendar.JANUARY` and :data:`calendar.FEBRUARY`. (Contributed by Prince Roshan in :gh:`103636`.) -* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. - Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. - For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. +* :mod:`collections.abc`: Deprecated :class:`collections.abc.ByteString`. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` implements + the :ref:`buffer protocol ` at runtime. For use in type + annotations, either use :class:`~collections.abc.Buffer` or a union + that explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was an + instance of :class:`!ByteString` never actually told you anything useful + about the object. Other common buffer types such as :class:`memoryview` were + also never understood as subtypes of :class:`!ByteString` (either at + runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. (Contributed by Shantanu Jain in :gh:`91896`.) * :mod:`datetime`: :class:`datetime.datetime`'s :meth:`~datetime.datetime.utcnow` and @@ -1301,7 +1315,7 @@ Deprecated :class:`collections.abc.Hashable` and :class:`collections.abc.Sized` respectively, are deprecated. (:gh:`94309`.) - * :class:`!typing.ByteString`, deprecated since Python 3.9, now causes a + * :class:`typing.ByteString`, deprecated since Python 3.9, now causes a :exc:`DeprecationWarning` to be emitted when it is used. (Contributed by Alex Waygood in :gh:`91896`.) @@ -1323,7 +1337,7 @@ Deprecated it was :exc:`ImportWarning`). (Contributed by Brett Cannon in :gh:`65961`.) -* Setting :attr:`~module.__package__` or :attr:`~module.__cached__` on a +* Setting :attr:`~module.__package__` or ``__cached__`` on a module is deprecated, and will cease to be set or taken into consideration by the import system in Python 3.14. (Contributed by Brett Cannon in :gh:`65961`.) @@ -1333,7 +1347,7 @@ Deprecated ``int``, convert to int explicitly: ``~int(x)``. (Contributed by Tim Hoffmann in :gh:`103487`.) -* Accessing :attr:`~codeobject.co_lnotab` on code objects was deprecated in +* Accessing :attr:`!codeobject.co_lnotab` on code objects was deprecated in Python 3.10 via :pep:`626`, but it only got a proper :exc:`DeprecationWarning` in 3.12. May be removed in 3.15. @@ -1349,6 +1363,12 @@ Deprecated .. include:: ../deprecations/pending-removal-in-3.17.rst +.. include:: ../deprecations/pending-removal-in-3.19.rst + +.. include:: ../deprecations/pending-removal-in-3.20.rst + +.. include:: ../deprecations/pending-removal-in-3.21.rst + .. include:: ../deprecations/pending-removal-in-future.rst .. _whatsnew312-removed: @@ -1729,7 +1749,7 @@ Changes in the Python API around process-global resources, which are best managed from the main interpreter. (Contributed by Donghee Na in :gh:`99127`.) -* The undocumented locking behavior of :func:`~functools.cached_property` +* The undocumented locking behavior of :deco:`~functools.cached_property` is removed, because it locked across all instances of the class, leading to high lock contention. This means that a cached property getter function could now run more than once for a single instance, if two threads race. For most simple diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 126329b538db314..d8845fe921ededa 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -60,7 +60,7 @@ Summary -- Release Highlights .. This section singles out the most important changes in Python 3.13. Brevity is key. -Python 3.13 is the latest stable release of the Python programming +Python 3.13 is a stable release of the Python programming language, with a mix of changes to the language, the implementation and the standard library. The biggest changes include a new `interactive interpreter @@ -834,7 +834,7 @@ dbm (Contributed by Raymond Hettinger and Erlend E. Aasland in :gh:`100414`.) * Allow removing all items from the database through - the new :meth:`.gdbm.clear` and :meth:`.ndbm.clear` methods. + the new :meth:`!clear` methods of the GDBM and NDBM database objects. (Contributed by Donghee Na in :gh:`107122`.) @@ -1569,8 +1569,6 @@ and are now removed: * :pypi:`bcrypt`: Modern password hashing for your software and your servers. - * :pypi:`passlib`: - Comprehensive password hashing framework supporting over 30 schemes. * :pypi:`argon2-cffi`: The secure Argon2 password hashing algorithm. * :pypi:`legacycrypt`: @@ -1863,6 +1861,7 @@ New Deprecations * Deprecate the non-standard and undocumented :class:`~decimal.Decimal` format specifier ``'N'``, which is only supported in the :mod:`!decimal` module's C implementation. + Scheduled to be removed in Python 3.18. (Contributed by Serhiy Storchaka in :gh:`89902`.) * :mod:`dis`: @@ -1959,14 +1958,14 @@ New Deprecations * :mod:`sys`: - * Deprecate the :func:`~sys._enablelegacywindowsfsencoding` function, + * Deprecate the :func:`!_enablelegacywindowsfsencoding` function, to be removed in Python 3.16. Use the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable instead. (Contributed by Inada Naoki in :gh:`73427`.) * :mod:`tarfile`: - * Deprecate the undocumented and unused :attr:`!TarFile.tarfile` attribute, + * Deprecate the undocumented and unused :attr:`!TarInfo.tarfile` attribute, to be removed in Python 3.16. (Contributed in :gh:`115256`.) @@ -1995,7 +1994,7 @@ New Deprecations use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. (Contributed by Alex Waygood in :gh:`105566` and :gh:`105570`.) - * Deprecate the :func:`typing.no_type_check_decorator` decorator function, + * Deprecate the :deco:`!typing.no_type_check_decorator` decorator function, to be removed in Python 3.15. After eight years in the :mod:`typing` module, it has yet to be supported by any major type checker. @@ -2026,6 +2025,14 @@ New Deprecations .. include:: ../deprecations/pending-removal-in-3.17.rst +.. include:: ../deprecations/pending-removal-in-3.18.rst + +.. include:: ../deprecations/pending-removal-in-3.19.rst + +.. include:: ../deprecations/pending-removal-in-3.20.rst + +.. include:: ../deprecations/pending-removal-in-3.21.rst + .. include:: ../deprecations/pending-removal-in-future.rst CPython Bytecode Changes @@ -2176,7 +2183,7 @@ New Features (Contributed by Sam Gross in :gh:`114329`.) * Add the :c:func:`PyList_Extend` and :c:func:`PyList_Clear` functions, - mirroring the Python :meth:`!list.extend` and :meth:`!list.clear` methods. + mirroring the Python :meth:`list.extend` and :meth:`list.clear` methods. (Contributed by Victor Stinner in :gh:`111138`.) * Add the :c:func:`PyLong_AsInt` function. @@ -2246,7 +2253,7 @@ New Features (Contributed by Serhiy Storchaka in :gh:`110289`.) * Add the :c:func:`PyWeakref_GetRef` function - as an alternative to :c:func:`PyWeakref_GetObject` + as an alternative to :c:func:`!PyWeakref_GetObject` that returns a :term:`strong reference` or ``NULL`` if the referent is no longer live. (Contributed by Victor Stinner in :gh:`105927`.) @@ -2335,7 +2342,7 @@ Limited C API Changes * :c:func:`PySys_AuditTuple` * :c:func:`PyType_GetModuleByDef` - (Contributed by Victor Stinner in :gh:`85283`, :gh:`85283`, and :gh:`116936`.) + (Contributed by Victor Stinner in :gh:`85283` and :gh:`116936`.) * Python built with :option:`--with-trace-refs` (tracing references) now supports the :ref:`Limited API `. @@ -2490,7 +2497,7 @@ Deprecated C APIs * Deprecate old Python initialization functions: - * :c:func:`PySys_ResetWarnOptions`: + * :c:func:`!PySys_ResetWarnOptions`: Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. * :c:func:`!Py_GetExecPrefix`: Get :data:`sys.exec_prefix` instead. @@ -2531,8 +2538,8 @@ Deprecated C APIs are just aliases to :c:type:`!wchar_t`. (Contributed by Victor Stinner in :gh:`105156`.) -* Deprecate the :c:func:`PyWeakref_GetObject` and - :c:func:`PyWeakref_GET_OBJECT` functions, +* Deprecate the :c:func:`!PyWeakref_GetObject` and + :c:func:`!PyWeakref_GET_OBJECT` functions, which return a :term:`borrowed reference`. Replace them with the new :c:func:`PyWeakref_GetRef` function, which returns a :term:`strong reference`. @@ -2592,7 +2599,7 @@ Build Changes .. _mimalloc library: https://github.com/microsoft/mimalloc/ -* The :file:`configure` option :option:`--with-system-libmpdec` +* The :file:`configure` option :option:`!--with-system-libmpdec` now defaults to ``yes``. The bundled copy of ``libmpdec`` will be removed in Python 3.16. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 44ee2bbeb7761f2..cd0d8b7cb006fee 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1,8 +1,9 @@ + **************************** What's new in Python 3.14 **************************** -:Editor: Hugo van Kemenade +:Editors: Adam Turner and Hugo van Kemenade .. Rules for maintenance: @@ -45,146 +46,194 @@ when researching a change. This article explains the new features in Python 3.14, compared to 3.13. - +Python 3.14 was released on 7 October 2025. For full details, see the :ref:`changelog `. .. seealso:: :pep:`745` -- Python 3.14 release schedule -.. note:: - - Prerelease users should be aware that this document is currently in draft - form. It will be updated substantially as Python 3.14 moves towards release, - so it's worth checking back even after reading earlier versions. - -Summary -- release highlights +Summary -- Release highlights ============================= .. This section singles out the most important changes in Python 3.14. Brevity is key. -Python 3.14 beta is the pre-release of the next version of the Python -programming language, with a mix of changes to the language, the -implementation and the standard library. - -The biggest changes to the implementation include template strings (:pep:`750`), -deferred evaluation of annotations (:pep:`649`), -and a new type of interpreter that uses tail calls. - -The library changes include the addition of a new :mod:`!annotationlib` module -for introspecting and wrapping annotations (:pep:`749`), -a new :mod:`!compression.zstd` module for Zstandard support (:pep:`784`), -plus syntax highlighting in the REPL, +Python 3.14 is the latest stable release of the Python programming +language, with a mix of changes to the language, the implementation, +and the standard library. +The biggest changes include :ref:`template string literals +`, +:ref:`deferred evaluation of annotations `, +and support for :ref:`subinterpreters ` in +the standard library. + +The library changes include significantly improved capabilities for +:ref:`introspection in asyncio `, +:ref:`support for Zstandard ` via a new +:mod:`compression.zstd` module, syntax highlighting in the REPL, as well as the usual deprecations and removals, and improvements in user-friendliness and correctness. +This article doesn't attempt to provide a complete specification +of all new features, but instead gives a convenient overview. +For full details refer to the documentation, +such as the :ref:`Library Reference ` +and :ref:`Language Reference `. +To understand the complete implementation and design rationale for a change, +refer to the PEP for a particular new feature; +but note that PEPs usually are not kept up-to-date +once a feature has been fully implemented. +See `Porting to Python 3.14`_ for guidance on upgrading from +earlier versions of Python. + +-------------- + .. PEP-sized items next. -* :ref:`PEP 779: Free-threaded Python is officially supported ` -* :ref:`PEP 649 and 749: deferred evaluation of annotations ` -* :ref:`PEP 734: Multiple interpreters in the stdlib ` -* :ref:`PEP 741: Python configuration C API ` -* :ref:`PEP 750: Template strings ` -* :ref:`PEP 758: Allow except and except* expressions without parentheses ` -* :ref:`PEP 761: Discontinuation of PGP signatures ` -* :ref:`PEP 765: Disallow return/break/continue that exit a finally block ` +Interpreter improvements: + +* :pep:`649` and :pep:`749`: :ref:`Deferred evaluation of annotations + ` +* :pep:`734`: :ref:`Multiple interpreters in the standard library + ` +* :pep:`750`: :ref:`Template strings ` +* :pep:`758`: :ref:`Allow except and except* expressions without brackets + ` +* :pep:`765`: :ref:`Control flow in finally blocks + ` +* :pep:`768`: :ref:`Safe external debugger interface for CPython + ` +* :ref:`A new type of interpreter ` * :ref:`Free-threaded mode improvements ` -* :ref:`PEP 768: Safe external debugger interface for CPython ` -* :ref:`PEP 784: Adding Zstandard to the standard library ` -* :ref:`A new type of interpreter ` -* :ref:`Syntax highlighting in PyREPL `, - and color output in :ref:`unittest `, - :ref:`argparse `, - :ref:`json ` and - :ref:`calendar ` CLIs -* :ref:`Binary releases for the experimental just-in-time compiler ` +* :ref:`Improved error messages ` +* :ref:`Incremental garbage collection ` + +Significant improvements in the standard library: + +* :pep:`784`: :ref:`Zstandard support in the standard library + ` +* :ref:`whatsnew314-asyncio-introspection` +* :ref:`whatsnew314-concurrent-warnings-control` +* :ref:`Syntax highlighting in the default interactive shell + `, and color output in several + standard library CLIs +C API improvements: -Incompatible changes -==================== +* :pep:`741`: :ref:`Python configuration C API ` -On platforms other than macOS and Windows, the default :ref:`start -method ` for :mod:`multiprocessing` -and :class:`~concurrent.futures.ProcessPoolExecutor` switches from -*fork* to *forkserver*. +Platform support: -See :ref:`(1) ` and -:ref:`(2) ` for details. +* :pep:`776`: Emscripten is now an :ref:`officially supported platform + `, at :pep:`tier 3 <11#tier-3>`. -If you encounter :exc:`NameError`\s or pickling errors coming out of -:mod:`multiprocessing` or :mod:`concurrent.futures`, see the -:ref:`forkserver restrictions `. +Release changes: + +* :pep:`779`: :ref:`Free-threaded Python is officially supported + ` +* :pep:`761`: :ref:`PGP signatures have been discontinued for official releases + ` +* :ref:`Windows and macOS binary releases now support the experimental + just-in-time compiler ` +* :ref:`Binary releases for Android are now provided ` -The interpreter avoids some reference count modifications internally when -it's safe to do so. This can lead to different values returned from -:func:`sys.getrefcount` and :c:func:`Py_REFCNT` compared to previous versions -of Python. See :ref:`below ` for details. New features ============ -.. _whatsnew314-pep779: +.. _whatsnew314-deferred-annotations: -PEP 779: Free-threaded Python is officially supported ------------------------------------------------------ +:pep:`649` & :pep:`749`: Deferred evaluation of annotations +------------------------------------------------------------ -The free-threaded build of Python is now supported and no longer experimental. -This is the start of phase II where free-threaded Python is officially supported -but still optional. +The :term:`annotations ` on functions, classes, and modules are no +longer evaluated eagerly. Instead, annotations are stored in special-purpose +:term:`annotate functions ` and evaluated only when +necessary (except if ``from __future__ import annotations`` is used). -We are confident that the project is on the right path, and we appreciate the -continued dedication from everyone working to make free-threading ready for -broader adoption across the Python community. +This change is designed to improve performance and usability of annotations +in Python in most circumstances. The runtime cost for defining annotations is +minimized, but it remains possible to introspect annotations at runtime. +It is no longer necessary to enclose annotations in strings if they +contain forward references. -With these recommendations and the acceptance of this PEP, we as the Python -developer community should broadly advertise that free-threading is a supported -Python build option now and into the future, and that it will not be removed -without a proper deprecation schedule. +The new :mod:`annotationlib` module provides tools for inspecting deferred +annotations. Annotations may be evaluated in the :attr:`~annotationlib.Format.VALUE` +format (which evaluates annotations to runtime values, similar to the behavior in +earlier Python versions), the :attr:`~annotationlib.Format.FORWARDREF` format +(which replaces undefined names with special markers), and the +:attr:`~annotationlib.Format.STRING` format (which returns annotations as strings). + +This example shows how these formats behave: -Any decision to transition to phase III, with free-threading as the default or -sole build of Python is still undecided, and dependent on many factors both -within CPython itself and the community. This decision is for the future. +.. doctest:: + + >>> from annotationlib import get_annotations, Format + >>> def func(arg: Undefined): + ... pass + >>> get_annotations(func, format=Format.VALUE) + Traceback (most recent call last): + ... + NameError: name 'Undefined' is not defined + >>> get_annotations(func, format=Format.FORWARDREF) + {'arg': ForwardRef('Undefined', owner=)} + >>> get_annotations(func, format=Format.STRING) + {'arg': 'Undefined'} + +The :ref:`porting ` section contains guidance +on changes that may be needed due to these changes, though in the majority of +cases, code will continue working as-is. + +(Contributed by Jelle Zijlstra in :pep:`749` and :gh:`119180`; +:pep:`649` was written by Larry Hastings.) .. seealso:: - :pep:`779` and its `acceptance - `__. -.. _whatsnew314-pep734: + :pep:`649` + Deferred Evaluation Of Annotations Using Descriptors + :pep:`749` + Implementing PEP 649 -PEP 734: Multiple interpreters in the stdlib --------------------------------------------- + +.. _whatsnew314-multiple-interpreters: + +:pep:`734`: Multiple interpreters in the standard library +--------------------------------------------------------- The CPython runtime supports running multiple copies of Python in the same process simultaneously and has done so for over 20 years. -Each of these separate copies is called an "interpreter". -However, the feature had been available only through the C-API. +Each of these separate copies is called an 'interpreter'. +However, the feature had been available only through +the :ref:`C-API `. -That limitation is removed in the 3.14 release, +That limitation is removed in Python 3.14, with the new :mod:`concurrent.interpreters` module. There are at least two notable reasons why using multiple interpreters -is worth considering: +has significant benefits: * they support a new (to Python), human-friendly concurrency model * true multi-core parallelism -For some use cases, concurrency in software enables efficiency and -can simplify software, at a high level. At the same time, implementing -and maintaining all but the simplest concurrency is often a struggle -for the human brain. That especially applies to plain threads -(for example, :mod:`threading`), where all memory is shared between all threads. +For some use cases, concurrency in software improves efficiency and +can simplify design, at a high level. +At the same time, implementing and maintaining all but the simplest concurrency +is often a struggle for the human brain. +That especially applies to plain threads (for example, :mod:`threading`), +where all memory is shared between all threads. With multiple isolated interpreters, you can take advantage of a class -of concurrency models, like CSP or the actor model, that have found +of concurrency models, like Communicating Sequential Processes (CSP) +or the actor model, that have found success in other programming languages, like Smalltalk, Erlang, -Haskell, and Go. Think of multiple interpreters like threads +Haskell, and Go. Think of multiple interpreters as threads but with opt-in sharing. -Regarding multi-core parallelism: as of the 3.12 release, interpreters -are now sufficiently isolated from one another to be used in parallel. -(See :pep:`684`.) This unlocks a variety of CPU-intensive use cases +Regarding multi-core parallelism: as of Python 3.12, interpreters +are now sufficiently isolated from one another to be used in parallel +(see :pep:`684`). This unlocks a variety of CPU-intensive use cases for Python that were limited by the :term:`GIL`. Using multiple interpreters is similar in many ways to @@ -192,402 +241,285 @@ Using multiple interpreters is similar in many ways to "processes" that can run in parallel, with no sharing by default. However, when using multiple interpreters, an application will use fewer system resources and will operate more efficiently (since it -stays within the same process). Think of multiple interpreters as +stays within the same process). Think of multiple interpreters as having the isolation of processes with the efficiency of threads. .. XXX Add an example or two. .. XXX Link to the not-yet-added HOWTO doc. While the feature has been around for decades, multiple interpreters -have not been used widely, due to low awareness and the lack of a stdlib -module. Consequently, they currently have several notable limitations, -which will improve significantly now that the feature is finally -going mainstream. +have not been used widely, due to low awareness and the lack of a +standard library module. Consequently, they currently have several +notable limitations, which are expected to improve significantly now +that the feature is going mainstream. Current limitations: * starting each interpreter has not been optimized yet * each interpreter uses more memory than necessary - (we will be working next on extensive internal sharing between - interpreters) + (work continues on extensive internal sharing between interpreters) * there aren't many options *yet* for truly sharing objects or other data between interpreters (other than :type:`memoryview`) -* many extension modules on PyPI are not compatible with multiple - interpreters yet (stdlib extension modules *are* compatible) +* many third-party extension modules on PyPI are not yet compatible + with multiple interpreters + (all standard library extension modules *are* compatible) * the approach to writing applications that use multiple isolated interpreters is mostly unfamiliar to Python users, for now The impact of these limitations will depend on future CPython improvements, how interpreters are used, and what the community solves -through PyPI packages. Depending on the use case, the limitations may +through PyPI packages. Depending on the use case, the limitations may not have much impact, so try it out! Furthermore, future CPython releases will reduce or eliminate overhead -and provide utilities that are less appropriate on PyPI. In the +and provide utilities that are less appropriate on PyPI. In the meantime, most of the limitations can also be addressed through extension modules, meaning PyPI packages can fill any gap for 3.14, and even back to 3.12 where interpreters were finally properly isolated and -stopped sharing the :term:`GIL`. Likewise, we expect to slowly see -libraries on PyPI for high-level abstractions on top of interpreters. +stopped sharing the :term:`GIL`. Likewise, libraries on PyPI are expected +to emerge for high-level abstractions on top of interpreters. Regarding extension modules, work is in progress to update some PyPI projects, as well as tools like Cython, pybind11, nanobind, and PyO3. The steps for isolating an extension module are found at -:ref:`isolating-extensions-howto`. Isolating a module has a lot of -overlap with what is required to support -:ref:`free-threading `, -so the ongoing work in the community in that area will help accelerate -support for multiple interpreters. +:ref:`isolating-extensions-howto`. +Isolating a module has a lot of overlap with what is required to support +:ref:`free-threading `, so the ongoing +work in the community in that area will help accelerate support +for multiple interpreters. Also added in 3.14: :ref:`concurrent.futures.InterpreterPoolExecutor `. -.. seealso:: - :pep:`734`. - +(Contributed by Eric Snow in :gh:`134939`.) -.. _whatsnew314-pep750: +.. seealso:: :pep:`734` -PEP 750: Template strings -------------------------- -Template string literals (t-strings) are a generalization of f-strings, -using a ``t`` in place of the ``f`` prefix. Instead of evaluating -to :class:`str`, t-strings evaluate to a new :class:`!string.templatelib.Template` type: +.. _whatsnew314-template-string-literals: -.. code-block:: python +:pep:`750`: Template string literals +------------------------------------ - from string.templatelib import Template +Template strings are a new mechanism for custom string processing. +They share the familiar syntax of f-strings but, unlike f-strings, +return an object representing the static and interpolated parts of +the string, instead of a simple :class:`str`. - name = "World" - template: Template = t"Hello {name}" +To write a t-string, use a ``'t'`` prefix instead of an ``'f'``: -The template can then be combined with functions that operate on the template's -structure to produce a :class:`str` or a string-like result. -For example, sanitizing input: +.. doctest:: -.. code-block:: python + >>> variety = 'Stilton' + >>> template = t'Try some {variety} cheese!' + >>> type(template) + - evil = "" - template = t"

{evil}

" - assert html(template) == "

<script>alert('evil')</script>

" +:class:`~string.templatelib.Template` objects provide access to the static +and interpolated (in curly braces) parts of a string *before* they are combined. +Iterate over :class:`!Template` instances to access their parts in order: -As another example, generating HTML attributes from data: +.. testsetup:: -.. code-block:: python + variety = 'Stilton' + template = t'Try some {variety} cheese!' - attributes = {"src": "shrubbery.jpg", "alt": "looks nice"} - template = t"" - assert html(template) == 'looks nice' +.. doctest:: -Compared to using an f-string, the ``html`` function has access to template attributes -containing the original information: static strings, interpolations, and values -from the original scope. Unlike existing templating approaches, t-strings build -from the well-known f-string syntax and rules. Template systems thus benefit -from Python tooling as they are much closer to the Python language, syntax, -scoping, and more. + >>> list(template) + ['Try some ', Interpolation('Stilton', 'variety', None, ''), ' cheese!'] -Writing template handlers is straightforward: +It's easy to write (or call) code to process :class:`!Template` instances. +For example, here's a function that renders static parts lowercase and +:class:`~string.templatelib.Interpolation` instances uppercase: .. code-block:: python - from string.templatelib import Template, Interpolation + from string.templatelib import Interpolation - def lower_upper(template: Template) -> str: - """Render static parts lowercased and interpolations uppercased.""" - parts: list[str] = [] - for item in template: - if isinstance(item, Interpolation): - parts.append(str(item.value).upper()) + def lower_upper(template): + """Render static parts lowercase and interpolations uppercase.""" + parts = [] + for part in template: + if isinstance(part, Interpolation): + parts.append(str(part.value).upper()) else: - parts.append(item.lower()) - return "".join(parts) - - name = "world" - assert lower_upper(t"HELLO {name}") == "hello WORLD" - -With this in place, developers can write template systems to sanitize SQL, make -safe shell operations, improve logging, tackle modern ideas in web development -(HTML, CSS, and so on), and implement lightweight, custom business DSLs. - -(Contributed by Jim Baker, Guido van Rossum, Paul Everitt, Koudai Aono, -Lysandros Nikolaou, Dave Peck, Adam Turner, Jelle Zijlstra, Bénédikt Tran, -and Pablo Galindo Salgado in :gh:`132661`.) + parts.append(part.lower()) + return ''.join(parts) -.. seealso:: - :pep:`750`. - - -.. _whatsnew314-pep768: - -PEP 768: Safe external debugger interface for CPython ------------------------------------------------------ - -:pep:`768` introduces a zero-overhead debugging interface that allows debuggers and profilers -to safely attach to running Python processes. This is a significant enhancement to Python's -debugging capabilities allowing debuggers to forego unsafe alternatives. See -:ref:`below ` for how this feature is leveraged to -implement the new :mod:`pdb` module's remote attaching capabilities. - -The new interface provides safe execution points for attaching debugger code without modifying -the interpreter's normal execution path or adding runtime overhead. This enables tools to -inspect and interact with Python applications in real-time without stopping or restarting -them — a crucial capability for high-availability systems and production environments. - -For convenience, CPython implements this interface through the :mod:`sys` module with a -:func:`sys.remote_exec` function:: - - sys.remote_exec(pid, script_path) - -This function allows sending Python code to be executed in a target process at the next safe -execution point. However, tool authors can also implement the protocol directly as described -in the PEP, which details the underlying mechanisms used to safely attach to running processes. - -Here's a simple example that inspects object types in a running Python process: - - .. code-block:: python - - import os - import sys - import tempfile - - # Create a temporary script - with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: - script_path = f.name - f.write(f"import my_debugger; my_debugger.connect({os.getpid()})") - try: - # Execute in process with PID 1234 - print("Behold! An offering:") - sys.remote_exec(1234, script_path) - finally: - os.unlink(script_path) - -The debugging interface has been carefully designed with security in mind and includes several -mechanisms to control access: - -* A :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` environment variable. -* A :option:`-X disable-remote-debug` command-line option. -* A :option:`--without-remote-debug` configure flag to completely disable the feature at build time. - -A key implementation detail is that the interface piggybacks on the interpreter's existing evaluation -loop and safe points, ensuring zero overhead during normal execution while providing a reliable way -for external processes to coordinate debugging operations. - -(Contributed by Pablo Galindo Salgado, Matt Wozniski, and Ivona Stojanovic in :gh:`131591`.) - -.. seealso:: - :pep:`768`. - - -.. _whatsnew314-pep784: - -PEP 784: Adding Zstandard to the standard library -------------------------------------------------- - -The new ``compression`` package contains modules :mod:`!compression.lzma`, -:mod:`!compression.bz2`, :mod:`!compression.gzip` and :mod:`!compression.zlib` -which re-export the :mod:`lzma`, :mod:`bz2`, :mod:`gzip` and :mod:`zlib` -modules respectively. The new import names under ``compression`` are the -canonical names for importing these compression modules going forward. However, -the existing modules names have not been deprecated. Any deprecation or removal -of the existing compression modules will occur no sooner than five years after -the release of 3.14. - -The new :mod:`!compression.zstd` module provides compression and decompression -APIs for the Zstandard format via bindings to `Meta's zstd library -`__. Zstandard is a widely adopted, highly -efficient, and fast compression format. In addition to the APIs introduced in -:mod:`!compression.zstd`, support for reading and writing Zstandard compressed -archives has been added to the :mod:`tarfile`, :mod:`zipfile`, and -:mod:`shutil` modules. + name = 'Wenslydale' + template = t'Mister {name}' + assert lower_upper(template) == 'mister WENSLYDALE' -Here's an example of using the new module to compress some data: +Because :class:`!Template` instances distinguish between static strings and +interpolations at runtime, they can be useful for sanitising user input. +Writing a :func:`!html` function that escapes user input in HTML is an exercise +left to the reader! +Template processing code can provide improved flexibility. +For instance, a more advanced :func:`!html` function could accept +a :class:`!dict` of HTML attributes directly in the template: .. code-block:: python - from compression import zstd - import math - - data = str(math.pi).encode() * 20 - - compressed = zstd.compress(data) - - ratio = len(compressed) / len(data) - print(f"Achieved compression ratio of {ratio}") - -As can be seen, the API is similar to the APIs of the :mod:`!lzma` and -:mod:`!bz2` modules. - -(Contributed by Emma Harper Smith, Adam Turner, Gregory P. Smith, Tomas Roun, -Victor Stinner, and Rogdham in :gh:`132983`.) - -.. seealso:: - :pep:`784`. - - -.. _whatsnew314-remote-pdb: - -Remote attaching to a running Python process with PDB ------------------------------------------------------ - -The :mod:`pdb` module now supports remote attaching to a running Python process -using a new ``-p PID`` command-line option: + attributes = {'src': 'limburger.jpg', 'alt': 'lovely cheese'} + template = t'' + assert html(template) == 'lovely cheese' -.. code-block:: sh +Of course, template processing code does not need to return a string-like result. +An even *more* advanced :func:`!html` could return a custom type representing +a DOM-like structure. - python -m pdb -p 1234 +With t-strings in place, developers can write systems that sanitise SQL, +make safe shell operations, improve logging, tackle modern ideas in web +development (HTML, CSS, and so on), and implement lightweight custom business DSLs. -This will connect to the Python process with the given PID and allow you to -debug it interactively. Notice that due to how the Python interpreter works -attaching to a remote process that is blocked in a system call or waiting for -I/O will only work once the next bytecode instruction is executed or when the -process receives a signal. - -This feature uses :pep:`768` and the :func:`sys.remote_exec` function -to attach to the remote process and send the PDB commands to it. +(Contributed by Jim Baker, Guido van Rossum, Paul Everitt, Koudai Aono, +Lysandros Nikolaou, Dave Peck, Adam Turner, Jelle Zijlstra, Bénédikt Tran, +and Pablo Galindo Salgado in :gh:`132661`.) +.. seealso:: :pep:`750`. -(Contributed by Matt Wozniski and Pablo Galindo in :gh:`131591`.) -.. seealso:: - :pep:`768`. +.. _whatsnew314-remote-debugging: +:pep:`768`: Safe external debugger interface +-------------------------------------------- -.. _whatsnew314-pep758: +Python 3.14 introduces a zero-overhead debugging interface that allows +debuggers and profilers to safely attach to running Python processes +without stopping or restarting them. +This is a significant enhancement to Python's debugging capabilities, +meaning that unsafe alternatives are no longer required. -PEP 758 – Allow except and except* expressions without parentheses ------------------------------------------------------------------- +The new interface provides safe execution points for attaching debugger code +without modifying the interpreter's normal execution path +or adding any overhead at runtime. +Due to this, tools can now inspect and interact with Python applications +in real-time, which is a crucial capability for high-availability systems +and production environments. -The :keyword:`except` and :keyword:`except* ` expressions now allow -parentheses to be omitted when there are multiple exception types and the ``as`` clause is not used. -For example the following expressions are now valid: +For convenience, this interface is implemented in the :func:`sys.remote_exec` +function. For example: .. code-block:: python - try: - connect_to_server() - except TimeoutError, ConnectionRefusedError: - print("Network issue encountered.") - - # The same applies to except* (for exception groups): - - try: - connect_to_server() - except* TimeoutError, ConnectionRefusedError: - print("Network issue encountered.") + import sys + from tempfile import NamedTemporaryFile -Check :pep:`758` for more details. + with NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + script_path = f.name + f.write(f'import my_debugger; my_debugger.connect({os.getpid()})') -(Contributed by Pablo Galindo and Brett Cannon in :gh:`131831`.) + # Execute in process with PID 1234 + print('Behold! An offering:') + sys.remote_exec(1234, script_path) -.. seealso:: - :pep:`758`. +This function allows sending Python code to be executed in a target process +at the next safe execution point. +However, tool authors can also implement the protocol directly as described +in the PEP, which details the underlying mechanisms used to safely attach to +running processes. -.. _whatsnew314-pep649: +The debugging interface has been carefully designed with security in mind +and includes several mechanisms to control access: -PEP 649 and 749: deferred evaluation of annotations ---------------------------------------------------- +* A :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` environment variable. +* A :option:`-X disable-remote-debug` command-line option. +* A :option:`--without-remote-debug` configure flag to completely disable + the feature at build time. -The :term:`annotations ` on functions, classes, and modules are no -longer evaluated eagerly. Instead, annotations are stored in special-purpose -:term:`annotate functions ` and evaluated only when -necessary (except if ``from __future__ import annotations`` is used). -This is specified in :pep:`649` and :pep:`749`. +(Contributed by Pablo Galindo Salgado, Matt Wozniski, and Ivona Stojanovic +in :gh:`131591`.) -This change is designed to make annotations in Python more performant and more -usable in most circumstances. The runtime cost for defining annotations is -minimized, but it remains possible to introspect annotations at runtime. -It is no longer necessary to enclose annotations in strings if they -contain forward references. +.. seealso:: :pep:`768`. -The new :mod:`annotationlib` module provides tools for inspecting deferred -annotations. Annotations may be evaluated in the :attr:`~annotationlib.Format.VALUE` -format (which evaluates annotations to runtime values, similar to the behavior in -earlier Python versions), the :attr:`~annotationlib.Format.FORWARDREF` format -(which replaces undefined names with special markers), and the -:attr:`~annotationlib.Format.STRING` format (which returns annotations as strings). -This example shows how these formats behave: +.. _whatsnew314-tail-call-interpreter: -.. doctest:: +A new type of interpreter +------------------------- - >>> from annotationlib import get_annotations, Format - >>> def func(arg: Undefined): - ... pass - >>> get_annotations(func, format=Format.VALUE) - Traceback (most recent call last): - ... - NameError: name 'Undefined' is not defined - >>> get_annotations(func, format=Format.FORWARDREF) - {'arg': ForwardRef('Undefined', owner=)} - >>> get_annotations(func, format=Format.STRING) - {'arg': 'Undefined'} +A new type of interpreter has been added to CPython. +It uses tail calls between small C functions that implement individual +Python opcodes, rather than one large C ``case`` statement. +For certain newer compilers, this interpreter provides +significantly better performance. Preliminary benchmarks suggest a geometric +mean of 3-5% faster on the standard ``pyperformance`` benchmark suite, +depending on platform and architecture. +The baseline is Python 3.14 built with Clang 19, without this new interpreter. -Implications for annotated code -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +This interpreter currently only works with Clang 19 and newer +on x86-64 and AArch64 architectures. +However, a future release of GCC is expected to support this as well. -If you define annotations in your code (for example, for use with a static type -checker), then this change probably does not affect you: you can keep -writing annotations the same way you did with previous versions of Python. +This feature is opt-in for now. Enabling profile-guided optimization is highly +recommended when using the new interpreter as it is the only configuration +that has been tested and validated for improved performance. +For further information, see :option:`--with-tail-call-interp`. -You will likely be able to remove quoted strings in annotations, which are frequently -used for forward references. Similarly, if you use ``from __future__ import annotations`` -to avoid having to write strings in annotations, you may well be able to -remove that import once you support only Python 3.14 and newer. -However, if you rely on third-party libraries that read annotations, -those libraries may need changes to support unquoted annotations before they -work as expected. +.. note:: -Implications for readers of ``__annotations__`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + This is not to be confused with `tail call optimization`__ of Python + functions, which is currently not implemented in CPython. -If your code reads the ``__annotations__`` attribute on objects, you may want -to make changes in order to support code that relies on deferred evaluation of -annotations. For example, you may want to use :func:`annotationlib.get_annotations` -with the :attr:`~annotationlib.Format.FORWARDREF` format, as the :mod:`dataclasses` -module now does. + This new interpreter type is an internal implementation detail of the CPython + interpreter. It doesn't change the visible behavior of Python programs at + all. It can improve their performance, but doesn't change anything else. -The external :pypi:`typing_extensions` package provides partial backports of some of the -functionality of the :mod:`annotationlib` module, such as the :class:`~annotationlib.Format` -enum and the :func:`~annotationlib.get_annotations` function. These can be used to -write cross-version code that takes advantage of the new behavior in Python 3.14. + __ https://en.wikipedia.org/wiki/Tail_call -Related changes -^^^^^^^^^^^^^^^ +(Contributed by Ken Jin in :gh:`128563`, with ideas on how to implement this +in CPython by Mark Shannon, Garrett Gu, Haoran Xu, and Josh Haberman.) -The changes in Python 3.14 are designed to rework how ``__annotations__`` -works at runtime while minimizing breakage to code that contains -annotations in source code and to code that reads ``__annotations__``. However, -if you rely on undocumented details of the annotation behavior or on private -functions in the standard library, there are many ways in which your code may -not work in Python 3.14. To safeguard your code against future changes, -use only the documented functionality of the :mod:`annotationlib` module. -In particular, do not read annotations directly from the namespace dictionary -attribute of type objects. Use :func:`annotationlib.get_annotate_from_class_namespace` -during class construction and :func:`annotationlib.get_annotations` afterwards. +.. _whatsnew314-free-threaded-cpython: -In previous releases, it was sometimes possible to access class annotations from -an instance of an annotated class. This behavior was undocumented and accidental, -and will no longer work in Python 3.14. +Free-threaded mode improvements +------------------------------- + +CPython's free-threaded mode (:pep:`703`), initially added in 3.13, +has been significantly improved in Python 3.14. +The implementation described in PEP 703 has been finished, including C API +changes, and temporary workarounds in the interpreter were replaced with +more permanent solutions. +The specializing adaptive interpreter (:pep:`659`) is now enabled +in free-threaded mode, which along with many other optimizations +greatly improves its performance. +The performance penalty on single-threaded code in free-threaded mode +is now roughly 5-10%, depending on the platform and C compiler used. + +From Python 3.14, when compiling extension modules for the free-threaded build of +CPython on Windows, the preprocessor variable ``Py_GIL_DISABLED`` now needs to +be specified by the build backend, as it will no longer be determined +automatically by the C compiler. For a running interpreter, the setting that +was used at compile time can be found using :func:`sysconfig.get_config_var`. -``from __future__ import annotations`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The new :option:`-X context_aware_warnings <-X>` flag controls if +:ref:`concurrent safe warnings control ` +is enabled. The flag defaults to true for the free-threaded build +and false for the GIL-enabled build. -In Python 3.7, :pep:`563` introduced the ``from __future__ import annotations`` -directive, which turns all annotations into strings. This directive is now -considered deprecated and it is expected to be removed in a future version of Python. -However, this removal will not happen until after Python 3.13, the last version of -Python without deferred evaluation of annotations, reaches its end of life in 2029. -In Python 3.14, the behavior of code using ``from __future__ import annotations`` -is unchanged. +A new :data:`~sys.flags.thread_inherit_context` flag has been added, +which if enabled means that threads created with :class:`threading.Thread` +start with a copy of the :class:`~contextvars.Context()` of the caller of +:meth:`~threading.Thread.start`. Most significantly, this makes the warning +filtering context established by :class:`~warnings.catch_warnings` be +"inherited" by threads (or asyncio tasks) started within that context. It also +affects other modules that use context variables, such as the :mod:`decimal` +context manager. +This flag defaults to true for the free-threaded build and false for +the GIL-enabled build. -(Contributed by Jelle Zijlstra in :gh:`119180`; :pep:`649` was written by Larry Hastings.) +(Contributed by Sam Gross, Matt Page, Neil Schemenauer, Thomas Wouters, +Donghee Na, Kirill Podoprigora, Ken Jin, Itamar Oren, Brett Simmers, +Dino Viehland, Nathan Goldbaum, Ralf Gommers, Lysandros Nikolaou, Kumar Aditya, +Edgar Margffoy, and many others. +Some of these contributors are employed by Meta, which has continued to provide +significant engineering resources to support this project.) -.. seealso:: - :pep:`649` and :pep:`749`. +.. _whatsnew314-improved-error-messages: Improved error messages ----------------------- @@ -608,47 +540,12 @@ Improved error messages ^^^^^^ SyntaxError: invalid syntax. Did you mean 'while'? - >>> asynch def fetch_data(): - ... pass - Traceback (most recent call last): - File "", line 1 - asynch def fetch_data(): - ^^^^^^ - SyntaxError: invalid syntax. Did you mean 'async'? - - >>> async def foo(): - ... awaid fetch_data() - Traceback (most recent call last): - File "", line 2 - awaid fetch_data() - ^^^^^ - SyntaxError: invalid syntax. Did you mean 'await'? - - >>> raisee ValueError("Error") - Traceback (most recent call last): - File "", line 1 - raisee ValueError("Error") - ^^^^^^ - SyntaxError: invalid syntax. Did you mean 'raise'? - While the feature focuses on the most common cases, some variations of misspellings may still result in regular syntax errors. (Contributed by Pablo Galindo in :gh:`132449`.) -* When unpacking assignment fails due to incorrect number of variables, the - error message prints the received number of values in more cases than before. - (Contributed by Tushar Sadhwani in :gh:`122239`.) - - .. code-block:: pycon - - >>> x, y, z = 1, 2, 3, 4 - Traceback (most recent call last): - File "", line 1, in - x, y, z = 1, 2, 3, 4 - ^^^^^^^ - ValueError: too many values to unpack (expected 3, got 4) - -* :keyword:`elif` statements that follow an :keyword:`else` block now have a specific error message. +* :keyword:`elif` statements that follow an :keyword:`else` block now have + a specific error message. (Contributed by Steele Farnsworth in :gh:`129902`.) .. code-block:: pycon @@ -664,11 +561,9 @@ Improved error messages ^^^^ SyntaxError: 'elif' block follows an 'else' block -* If a statement (:keyword:`pass`, :keyword:`del`, :keyword:`return`, - :keyword:`yield`, :keyword:`raise`, :keyword:`break`, :keyword:`continue`, - :keyword:`assert`, :keyword:`import`, :keyword:`from`) is passed to the - :ref:`if_expr` after :keyword:`else`, or one of :keyword:`pass`, - :keyword:`break`, or :keyword:`continue` is passed before :keyword:`if`, then the +* If a statement is passed to the :ref:`if_expr` after :keyword:`else`, + or one of :keyword:`pass`, :keyword:`break`, or :keyword:`continue` + is passed before :keyword:`if`, then the error message highlights where the :token:`~python-grammar:expression` is required. (Contributed by Sergey Miryanov in :gh:`129515`.) @@ -688,10 +583,9 @@ Improved error messages ^^^^^^^^ SyntaxError: expected expression before 'if', but statement is given - * When incorrectly closed strings are detected, the error message suggests - that the string may be intended to be part of the string. (Contributed by - Pablo Galindo in :gh:`88535`.) + that the string may be intended to be part of the string. + (Contributed by Pablo Galindo in :gh:`88535`.) .. code-block:: pycon @@ -700,8 +594,8 @@ Improved error messages SyntaxError: invalid syntax. Is this intended to be part of the string? * When strings have incompatible prefixes, the error now shows - which prefixes are incompatible. (Contributed by - Nikita Sobolev in :gh:`133197`.) + which prefixes are incompatible. + (Contributed by Nikita Sobolev in :gh:`133197`.) .. code-block:: pycon @@ -718,20 +612,11 @@ Improved error messages - Except handlers: ``except ... as ...`` - Pattern-match cases: ``case ... as ...`` - (Contributed by Nikita Sobolev in :gh:`123539`, - :gh:`123562`, and :gh:`123440`.) - - .. code-block:: pycon - - >>> import ast as arr[0] - File "", line 1 - import ast as arr[0] - ^^^^^^ - SyntaxError: cannot use subscript as import target + (Contributed by Nikita Sobolev in :gh:`123539`, :gh:`123562`, and :gh:`123440`.) * Improved error message when trying to add an instance of an unhashable type to - a :class:`dict` or :class:`set`. (Contributed by CF Bolz-Tereick and Victor Stinner - in :gh:`132828`.) + a :class:`dict` or :class:`set`. + (Contributed by CF Bolz-Tereick and Victor Stinner in :gh:`132828`.) .. code-block:: pycon @@ -751,66 +636,77 @@ Improved error messages ~^^^ TypeError: cannot use 'list' as a dict key (unhashable type: 'list') +* Improved error message when an object supporting the synchronous + context manager protocol is entered using :keyword:`async with` + instead of :keyword:`with`, + and vice versa for the asynchronous context manager protocol. + (Contributed by Bénédikt Tran in :gh:`128398`.) -.. _whatsnew314-pep741: - -PEP 741: Python configuration C API ------------------------------------ -Add a :ref:`PyInitConfig C API ` to configure the Python -initialization without relying on C structures and the ability to make -ABI-compatible changes in the future. +.. _whatsnew314-zstandard: -Complete the :pep:`587` :ref:`PyConfig C API ` by adding -:c:func:`PyInitConfig_AddModule` which can be used to add a built-in extension -module; feature previously referred to as the “inittab”. +:pep:`784`: Zstandard support in the standard library +----------------------------------------------------- -Add :c:func:`PyConfig_Get` and :c:func:`PyConfig_Set` functions to get and set -the current runtime configuration. +The new :mod:`!compression` package contains modules :mod:`!compression.lzma`, +:mod:`!compression.bz2`, :mod:`!compression.gzip` and :mod:`!compression.zlib` +which re-export the :mod:`lzma`, :mod:`bz2`, :mod:`gzip` and :mod:`zlib` +modules respectively. The new import names under :mod:`!compression` are the +preferred names for importing these compression modules from Python 3.14. However, +the existing modules names have not been deprecated. Any deprecation or removal +of the existing compression modules will occur no sooner than five years after +the release of 3.14. -PEP 587 “Python Initialization Configuration” unified all the ways to configure -the Python initialization. This PEP unifies also the configuration of the -Python preinitialization and the Python initialization in a single API. -Moreover, this PEP only provides a single choice to embed Python, instead of -having two “Python” and “Isolated” choices (PEP 587), to simplify the API -further. +The new :mod:`!compression.zstd` module provides compression and decompression +APIs for the Zstandard format via bindings to `Meta's zstd library +`__. Zstandard is a widely adopted, highly +efficient, and fast compression format. In addition to the APIs introduced in +:mod:`!compression.zstd`, support for reading and writing Zstandard compressed +archives has been added to the :mod:`tarfile`, :mod:`zipfile`, and +:mod:`shutil` modules. -The lower level PEP 587 PyConfig API remains available for use cases with an -intentionally higher level of coupling to CPython implementation details (such -as emulating the full functionality of CPython’s CLI, including its -configuration mechanisms). +Here's an example of using the new module to compress some data: -(Contributed by Victor Stinner in :gh:`107954`.) +.. code-block:: python -.. seealso:: - :pep:`741`. + from compression import zstd + import math -.. _whatsnew314-asyncio-introspection: + data = str(math.pi).encode() * 20 + compressed = zstd.compress(data) + ratio = len(compressed) / len(data) + print(f"Achieved compression ratio of {ratio}") -Asyncio introspection capabilities ----------------------------------- +As can be seen, the API is similar to the APIs of the :mod:`!lzma` and +:mod:`!bz2` modules. + +(Contributed by Emma Harper Smith, Adam Turner, Gregory P. Smith, Tomas Roun, +Victor Stinner, and Rogdham in :gh:`132983`.) -Added a new command-line interface to inspect running Python processes using -asynchronous tasks, available via: +.. seealso:: :pep:`784`. -.. code-block:: bash - python -m asyncio ps PID +.. _whatsnew314-asyncio-introspection: + +Asyncio introspection capabilities +---------------------------------- -This tool inspects the given process ID (PID) and displays information about -currently running asyncio tasks. It outputs a task table: a flat -listing of all tasks, their names, their coroutine stacks, and which tasks are -awaiting them. +Added a new command-line interface to inspect running Python processes +using asynchronous tasks, available via ``python -m asyncio ps PID`` +or ``python -m asyncio pstree PID``. -.. code-block:: bash +The ``ps`` subcommand inspects the given process ID (PID) and displays +information about currently running asyncio tasks. +It outputs a task table: a flat listing of all tasks, their names, +their coroutine stacks, and which tasks are awaiting them. - python -m asyncio pstree PID -This tool fetches the same information, but renders a visual async call tree, -showing coroutine relationships in a hierarchical format. This command is -particularly useful for debugging long-running or stuck asynchronous programs. -It can help developers quickly identify where a program is blocked, what tasks -are pending, and how coroutines are chained together. +The ``pstree`` subcommand fetches the same information, but instead renders a +visual async call tree, showing coroutine relationships in a hierarchical format. +This command is particularly useful for debugging long-running or stuck +asynchronous programs. +It can help developers quickly identify where a program is blocked, +what tasks are pending, and how coroutines are chained together. For example given this code: @@ -818,23 +714,25 @@ For example given this code: import asyncio - async def play(track): + async def play_track(track): await asyncio.sleep(5) - print(f"🎵 Finished: {track}") + print(f'🎵 Finished: {track}') - async def album(name, tracks): + async def play_album(name, tracks): async with asyncio.TaskGroup() as tg: for track in tracks: - tg.create_task(play(track), name=track) + tg.create_task(play_track(track), name=track) async def main(): async with asyncio.TaskGroup() as tg: tg.create_task( - album("Sundowning", ["TNDNBTG", "Levitate"]), name="Sundowning") + play_album('Sundowning', ['TNDNBTG', 'Levitate']), + name='Sundowning') tg.create_task( - album("TMBTE", ["DYWTYLM", "Aqua Regia"]), name="TMBTE") + play_album('TMBTE', ['DYWTYLM', 'Aqua Regia']), + name='TMBTE') - if __name__ == "__main__": + if __name__ == '__main__': asyncio.run(main()) Executing the new tool on the running process will yield a table like this: @@ -899,169 +797,180 @@ prevent tree construction: (Contributed by Pablo Galindo, Łukasz Langa, Yury Selivanov, and Marta Gomez Macias in :gh:`91048`.) -.. _whatsnew314-tail-call: -A new type of interpreter -------------------------- +.. _whatsnew314-concurrent-warnings-control: -A new type of interpreter has been added to CPython. -It uses tail calls between small C functions that implement individual -Python opcodes, rather than one large C case statement. -For certain newer compilers, this interpreter provides -significantly better performance. Preliminary numbers on our machines suggest -anywhere up to 30% faster Python code, and a geometric mean of 3-5% -faster on ``pyperformance`` depending on platform and architecture. The -baseline is Python 3.14 built with Clang 19 without this new interpreter. +Concurrent safe warnings control +-------------------------------- -This interpreter currently only works with Clang 19 and newer -on x86-64 and AArch64 architectures. However, we expect -that a future release of GCC will support this as well. +The :class:`warnings.catch_warnings` context manager will now optionally +use a context variable for warning filters. This is enabled by setting +the :data:`~sys.flags.context_aware_warnings` flag, either with the ``-X`` +command-line option or an environment variable. This gives predictable +warnings control when using :class:`~warnings.catch_warnings` combined with +multiple threads or asynchronous tasks. The flag defaults to true for the +free-threaded build and false for the GIL-enabled build. -This feature is opt-in for now. We highly recommend enabling profile-guided -optimization with the new interpreter as it is the only configuration we have -tested and can validate its improved performance. -For further information on how to build Python, see -:option:`--with-tail-call-interp`. +(Contributed by Neil Schemenauer and Kumar Aditya in :gh:`130010`.) -.. note:: - This is not to be confused with `tail call optimization`__ of Python - functions, which is currently not implemented in CPython. +Other language changes +====================== - This new interpreter type is an internal implementation detail of the CPython - interpreter. It doesn't change the visible behavior of Python programs at - all. It can improve their performance, but doesn't change anything else. +* All Windows code pages are now supported as 'cpXXX' codecs on Windows. + (Contributed by Serhiy Storchaka in :gh:`123803`.) - __ https://en.wikipedia.org/wiki/Tail_call +* Implement mixed-mode arithmetic rules combining real and complex numbers + as specified by the C standard since C99. + (Contributed by Sergey B Kirpichev in :gh:`69639`.) -.. attention:: +* More syntax errors are now detected regardless of optimisation and + the :option:`-O` command-line option. + This includes writes to ``__debug__``, incorrect use of :keyword:`await`, + and asynchronous comprehensions outside asynchronous functions. + For example, ``python -O -c 'assert (__debug__ := 1)'`` + or ``python -O -c 'assert await 1'`` now produce :exc:`SyntaxError`\ s. + (Contributed by Irit Katriel and Jelle Zijlstra in :gh:`122245` & :gh:`121637`.) - This section previously reported a 9-15% geometric mean speedup. This number has since been - cautiously revised down to 3-5%. While we expect performance results to be better - than what we report, our estimates are more conservative due to a - `compiler bug `_ found in - Clang/LLVM 19, which causes the normal interpreter to be slower. We were unaware of this bug, - resulting in inaccurate results. We sincerely apologize for - communicating results that were only accurate for LLVM v19.1.x and v20.1.0. In the meantime, - the bug has been fixed in LLVM v20.1.1 and for the upcoming v21.1, but it will remain - unfixed for LLVM v19.1.x and v20.1.0. Thus - any benchmarks with those versions of LLVM may produce inaccurate numbers. - (Thanks to Nelson Elhage for bringing this to light.) +* When subclassing a pure C type, the C slots for the new type + are no longer replaced with a wrapped version on class creation + if they are not explicitly overridden in the subclass. + (Contributed by Tomasz Pytel in :gh:`132284`.) -(Contributed by Ken Jin in :gh:`128563`, with ideas on how to implement this -in CPython by Mark Shannon, Garrett Gu, Haoran Xu, and Josh Haberman.) -.. _whatsnew314-free-threaded-cpython: +Built-ins +--------- -Free-threaded mode ------------------- +* The :meth:`bytes.fromhex` and :meth:`bytearray.fromhex` methods now accept + ASCII :class:`bytes` and :term:`bytes-like objects `. + (Contributed by Daniel Pope in :gh:`129349`.) -Free-threaded mode (:pep:`703`), initially added in 3.13, has been significantly improved. -The implementation described in PEP 703 was finished, including C API changes, -and temporary workarounds in the interpreter were replaced with more permanent solutions. -The specializing adaptive interpreter (:pep:`659`) is now enabled in free-threaded mode, -which along with many other optimizations greatly improves its performance. -The performance penalty on single-threaded code in free-threaded mode is now roughly 5-10%, -depending on platform and C compiler used. +* Add class methods :meth:`float.from_number` and :meth:`complex.from_number` + to convert a number to :class:`float` or :class:`complex` type correspondingly. + They raise a :exc:`TypeError` if the argument is not a real number. + (Contributed by Serhiy Storchaka in :gh:`84978`.) -This work was done by many contributors: Sam Gross, Matt Page, Neil Schemenauer, -Thomas Wouters, Donghee Na, Kirill Podoprigora, Ken Jin, Itamar Oren, -Brett Simmers, Dino Viehland, Nathan Goldbaum, Ralf Gommers, Lysandros Nikolaou, -Kumar Aditya, Edgar Margffoy, and many others. +* Support underscore and comma as thousands separators in the fractional part + for floating-point presentation types of the new-style string formatting + (with :func:`format` or :ref:`f-strings`). + (Contributed by Sergey B Kirpichev in :gh:`87790`.) -Some of these contributors are employed by Meta, which has continued to provide -significant engineering resources to support this project. +* The :func:`int` function no longer delegates to :meth:`~object.__trunc__`. + Classes that want to support conversion to :func:`!int` must implement + either :meth:`~object.__int__` or :meth:`~object.__index__`. + (Contributed by Mark Dickinson in :gh:`119743`.) -From 3.14, when compiling extension modules for the free-threaded build of -CPython on Windows, the preprocessor variable ``Py_GIL_DISABLED`` now needs to -be specified by the build backend, as it will no longer be determined -automatically by the C compiler. For a running interpreter, the setting that -was used at compile time can be found using :func:`sysconfig.get_config_var`. +* The :func:`map` function now has an optional keyword-only *strict* flag + like :func:`zip` to check that all the iterables are of equal length. + (Contributed by Wannes Boeykens in :gh:`119793`.) -A new flag has been added, :data:`~sys.flags.context_aware_warnings`. This -flag defaults to true for the free-threaded build and false for the GIL-enabled -build. If the flag is true then the :class:`warnings.catch_warnings` context -manager uses a context variable for warning filters. This makes the context -manager behave predicably when used with multiple threads or asynchronous -tasks. +* The :class:`memoryview` type now supports subscription, + making it a :term:`generic type`. + (Contributed by Brian Schubert in :gh:`126012`.) -A new flag has been added, :data:`~sys.flags.thread_inherit_context`. This flag -defaults to true for the free-threaded build and false for the GIL-enabled -build. If the flag is true then threads created with :class:`threading.Thread` -start with a copy of the :class:`~contextvars.Context()` of the caller of -:meth:`~threading.Thread.start`. Most significantly, this makes the warning -filtering context established by :class:`~warnings.catch_warnings` be -"inherited" by threads (or asyncio tasks) started within that context. It also -affects other modules that use context variables, such as the :mod:`decimal` -context manager. +* Using :data:`NotImplemented` in a boolean context + will now raise a :exc:`TypeError`. + This has raised a :exc:`DeprecationWarning` since Python 3.9. + (Contributed by Jelle Zijlstra in :gh:`118767`.) +* Three-argument :func:`pow` now tries calling :meth:`~object.__rpow__` + if necessary. + Previously it was only called in two-argument :func:`!pow` + and the binary power operator. + (Contributed by Serhiy Storchaka in :gh:`130104`.) -.. _whatsnew314-pyrepl-highlighting: +* :class:`super` objects are now :mod:`copyable ` and :mod:`pickleable + `. + (Contributed by Serhiy Storchaka in :gh:`125767`.) -Syntax highlighting in PyREPL ------------------------------ -The default :term:`interactive` shell now highlights Python syntax as you -type. The feature is enabled by default unless the -:envvar:`PYTHON_BASIC_REPL` environment is set or any color-disabling -environment variables are used. See :ref:`using-on-controlling-color` for -details. +Command line and environment +---------------------------- -The default color theme for syntax highlighting strives for good contrast -and uses exclusively the 4-bit VGA standard ANSI color codes for maximum -compatibility. The theme can be customized using an experimental API -``_colorize.set_theme()``. This can be called interactively, as well as -in the :envvar:`PYTHONSTARTUP` script. +* The import time flag can now track modules that are already loaded ('cached'), + via the new :option:`-X importtime=2 <-X>`. + When such a module is imported, the ``self`` and ``cumulative`` times + are replaced by the string ``cached``. -(Contributed by Łukasz Langa in :gh:`131507`.) + Values above ``2`` for ``-X importtime`` are now reserved for future use. + (Contributed by Noah Kim and Adam Turner in :gh:`118655`.) -.. _whatsnew314-jit-compiler: +* The command-line option :option:`-c` now automatically dedents its code + argument before execution. + (Contributed by Jon Crall and Steven Sun in :gh:`103998`.) -Binary releases for the experimental just-in-time compiler ----------------------------------------------------------- +* :option:`!-J` is no longer a reserved flag for Jython_, + and now has no special meaning. + (Contributed by Adam Turner in :gh:`133336`.) -The official macOS and Windows release binaries now include an *experimental* -just-in-time (JIT) compiler. Although it is **not** recommended for production -use, it can be tested by setting :envvar:`PYTHON_JIT=1 ` as an -environment variable. Downstream source builds and redistributors can use the -:option:`--enable-experimental-jit=yes-off` configuration option for similar -behavior. + .. _Jython: https://www.jython.org/ -The JIT is at an early stage and still in active development. As such, the -typical performance impact of enabling it can range from 10% slower to 20% -faster, depending on workload. To aid in testing and evaluation, a set of -introspection functions has been provided in the :data:`sys._jit` namespace. -:func:`sys._jit.is_available` can be used to determine if the current executable -supports JIT compilation, while :func:`sys._jit.is_enabled` can be used to tell -if JIT compilation has been enabled for the current process. -Currently, the most significant missing functionality is that native debuggers -and profilers like ``gdb`` and ``perf`` are unable to unwind through JIT frames -(Python debuggers and profilers, like :mod:`pdb` or :mod:`profile`, continue to -work without modification). Free-threaded builds do not support JIT compilation. +.. _whatsnew314-bracketless-except: -Please report any bugs or major performance regressions that you encounter! +PEP 758: Allow ``except`` and ``except*`` expressions without brackets +---------------------------------------------------------------------- -.. seealso:: :pep:`744` +The :keyword:`except` and :keyword:`except* ` expressions +now allow brackets to be omitted when there are multiple exception types +and the ``as`` clause is not used. +For example: -Concurrent safe warnings control --------------------------------- +.. code-block:: python -The :class:`warnings.catch_warnings` context manager will now optionally -use a context variable for warning filters. This is enabled by setting -the :data:`~sys.flags.context_aware_warnings` flag, either with the ``-X`` -command-line option or an environment variable. This gives predicable -warnings control when using :class:`~warnings.catch_warnings` combined with -multiple threads or asynchronous tasks. The flag defaults to true for the -free-threaded build and false for the GIL-enabled build. + try: + connect_to_server() + except TimeoutError, ConnectionRefusedError: + print('The network has ceased to be!') -(Contributed by Neil Schemenauer and Kumar Aditya in :gh:`130010`.) +(Contributed by Pablo Galindo and Brett Cannon in :pep:`758` and :gh:`131831`.) + + +.. _whatsnew314-finally-syntaxwarning: + +PEP 765: Control flow in :keyword:`finally` blocks +-------------------------------------------------- + +The compiler now emits a :exc:`SyntaxWarning` when a :keyword:`return`, +:keyword:`break`, or :keyword:`continue` statement have the effect of +leaving a :keyword:`finally` block. +This change is specified in :pep:`765`. + +In situations where this change is inconvenient (such as those where the +warnings are redundant due to code linting), the :ref:`warning filter +` can be used to turn off all syntax warnings by adding +``ignore::SyntaxWarning`` as a filter. This can be specified in combination +with a filter that converts other warnings to errors (for example, passing +``-Werror -Wignore::SyntaxWarning`` as CLI options, or setting +``PYTHONWARNINGS=error,ignore::SyntaxWarning``). + +Note that applying such a filter at runtime using the :mod:`warnings` module +will only suppress the warning in code that is compiled *after* the filter is +adjusted. Code that is compiled prior to the filter adjustment (for example, +when a module is imported) will still emit the syntax warning. + +(Contributed by Irit Katriel in :gh:`130080`.) + +.. _incremental-garbage-collection: .. _whatsnew314-incremental-gc: -Incremental garbage collection ------------------------------- +Garbage collection +------------------ + +**From Python 3.14.5 onwards:** + +The garbage collector (GC) has changed in Python 3.14.5. + +Python 3.14.0-3.14.4 shipped with a new incremental GC. +However, due to a number of `reports +`__ +of significant memory pressure in production environments, +it has been reverted back to the generational GC from 3.13. +This is the GC now used in Python 3.14.5 and later. + +**Previously in Python 3.14.0-3.14.4:** The cycle garbage collector is now incremental. This means that maximum pause times are reduced @@ -1082,133 +991,61 @@ The behavior of :func:`!gc.collect` changes slightly: (Contributed by Mark Shannon in :gh:`108362`.) -Other language changes -====================== - -* The default :term:`interactive` shell now supports import autocompletion. - This means that typing ``import foo`` and pressing ```` will suggest - modules starting with ``foo``. Similarly, typing ``from foo import b`` will - suggest submodules of ``foo`` starting with ``b``. Note that autocompletion - of module attributes is not currently supported. - (Contributed by Tomas Roun in :gh:`69605`.) - -* The :func:`map` built-in now has an optional keyword-only *strict* flag - like :func:`zip` to check that all the iterables are of equal length. - (Contributed by Wannes Boeykens in :gh:`119793`.) - -* Incorrect usage of :keyword:`await` and asynchronous comprehensions - is now detected even if the code is optimized away by the :option:`-O` - command-line option. For example, ``python -O -c 'assert await 1'`` - now produces a :exc:`SyntaxError`. (Contributed by Jelle Zijlstra in :gh:`121637`.) - -* Writes to ``__debug__`` are now detected even if the code is optimized - away by the :option:`-O` command-line option. For example, - ``python -O -c 'assert (__debug__ := 1)'`` now produces a - :exc:`SyntaxError`. (Contributed by Irit Katriel in :gh:`122245`.) - -* Add class methods :meth:`float.from_number` and :meth:`complex.from_number` - to convert a number to :class:`float` or :class:`complex` type correspondingly. - They raise an error if the argument is a string. - (Contributed by Serhiy Storchaka in :gh:`84978`.) - -* Implement mixed-mode arithmetic rules combining real and complex numbers as - specified by C standards since C99. - (Contributed by Sergey B Kirpichev in :gh:`69639`.) - -* All Windows code pages are now supported as "cpXXX" codecs on Windows. - (Contributed by Serhiy Storchaka in :gh:`123803`.) - -* :class:`super` objects are now :mod:`pickleable ` and - :mod:`copyable `. - (Contributed by Serhiy Storchaka in :gh:`125767`.) - -* The :class:`memoryview` type now supports subscription, - making it a :term:`generic type`. - (Contributed by Brian Schubert in :gh:`126012`.) - -* Support underscore and comma as thousands separators in the fractional part - for floating-point presentation types of the new-style string formatting - (with :func:`format` or :ref:`f-strings`). - (Contributed by Sergey B Kirpichev in :gh:`87790`.) - -* The :func:`bytes.fromhex` and :func:`bytearray.fromhex` methods now accept - ASCII :class:`bytes` and :term:`bytes-like objects `. - (Contributed by Daniel Pope in :gh:`129349`.) - -* Support ``\z`` as a synonym for ``\Z`` in :mod:`regular expressions `. - It is interpreted unambiguously in many other regular expression engines, - unlike ``\Z``, which has subtly different behavior. - (Contributed by Serhiy Storchaka in :gh:`133306`.) - -* ``\B`` in :mod:`regular expression ` now matches empty input string. - Now it is always the opposite of ``\b``. - (Contributed by Serhiy Storchaka in :gh:`124130`.) - -* iOS and macOS apps can now be configured to redirect ``stdout`` and - ``stderr`` content to the system log. (Contributed by Russell Keith-Magee in - :gh:`127592`.) - -* The iOS testbed is now able to stream test output while the test is running. - The testbed can also be used to run the test suite of projects other than - CPython itself. (Contributed by Russell Keith-Magee in :gh:`127592`.) - -* Three-argument :func:`pow` now tries calling :meth:`~object.__rpow__` if - necessary. Previously it was only called in two-argument :func:`!pow` and the - binary power operator. - (Contributed by Serhiy Storchaka in :gh:`130104`.) - -* Add a built-in implementation for HMAC (:rfc:`2104`) using formally verified - code from the `HACL* `__ project. - This implementation is used as a fallback when the OpenSSL implementation - of HMAC is not available. - (Contributed by Bénédikt Tran in :gh:`99108`.) - -* The import time flag can now track modules that are already loaded ('cached'), - via the new :option:`-X importtime=2 <-X>`. - When such a module is imported, the ``self`` and ``cumulative`` times - are replaced by the string ``cached``. - Values above ``2`` for ``-X importtime`` are now reserved for future use. - (Contributed by Noah Kim and Adam Turner in :gh:`118655`.) - -* When subclassing from a pure C type, the C slots for the new type are no - longer replaced with a wrapped version on class creation if they are not - explicitly overridden in the subclass. - (Contributed by Tomasz Pytel in :gh:`132329`.) - -* The command-line option :option:`-c` now automatically dedents its code - argument before execution. The auto-dedentation behavior mirrors - :func:`textwrap.dedent`. - (Contributed by Jon Crall and Steven Sun in :gh:`103998`.) - -* Improve error message when an object supporting the synchronous - context manager protocol is entered using :keyword:`async - with` instead of :keyword:`with`. - And vice versa with the asynchronous context manager protocol. - (Contributed by Bénédikt Tran in :gh:`128398`.) - -* :option:`!-J` is no longer a reserved flag for Jython_, - and now has no special meaning. - (Contributed by Adam Turner in :gh:`133336`.) - - .. _Jython: https://www.jython.org/ - -.. _whatsnew314-pep765: +Default interactive shell +------------------------- -PEP 765: Disallow ``return``/``break``/``continue`` that exit a ``finally`` block ---------------------------------------------------------------------------------- +.. _whatsnew314-pyrepl-highlighting: -The compiler emits a :exc:`SyntaxWarning` when a :keyword:`return`, :keyword:`break` or -:keyword:`continue` statements appears where it exits a :keyword:`finally` block. -This change is specified in :pep:`765`. +* The default :term:`interactive` shell now highlights Python syntax. + The feature is enabled by default, save if :envvar:`PYTHON_BASIC_REPL` + or any other environment variable that disables colour is set. + See :ref:`using-on-controlling-color` for details. + + The default color theme for syntax highlighting strives for good contrast + and exclusively uses the 4-bit VGA standard ANSI color codes for maximum + compatibility. The theme can be customized using an experimental API + :func:`!_colorize.set_theme`. + This can be called interactively or in the :envvar:`PYTHONSTARTUP` script. + Note that this function has no stability guarantees, + and may change or be removed. + + (Contributed by Łukasz Langa in :gh:`131507`.) + +* The default :term:`interactive` shell now supports import auto-completion. + This means that typing ``import co`` and pressing :kbd:`` will suggest + modules starting with ``co``. Similarly, typing ``from concurrent import i`` + will suggest submodules of ``concurrent`` starting with ``i``. + Note that autocompletion of module attributes is not currently supported. + (Contributed by Tomas Roun in :gh:`69605`.) New modules =========== -* :mod:`annotationlib`: For introspecting :term:`annotations `. - See :pep:`749` for more details. +* :mod:`annotationlib`: + For introspecting :term:`annotations `. + See :ref:`PEP 749 ` for more details. (Contributed by Jelle Zijlstra in :gh:`119180`.) +* :mod:`compression` (including :mod:`compression.zstd`): + A package for compression-related modules, + including a new module to support the Zstandard compression format. + See :ref:`PEP 784 ` for more details. + (Contributed by Emma Harper Smith, Adam Turner, Gregory P. Smith, Tomas Roun, + Victor Stinner, and Rogdham in :gh:`132983`.) + +* :mod:`concurrent.interpreters`: + Support for multiple interpreters in the standard library. + See :ref:`PEP 734 ` for more details. + (Contributed by Eric Snow in :gh:`134939`.) + +* :mod:`string.templatelib`: + Support for template string literals (t-strings). + See :ref:`PEP 750 ` for more details. + (Contributed by Jim Baker, Guido van Rossum, Paul Everitt, Koudai Aono, + Lysandros Nikolaou, Dave Peck, Adam Turner, Jelle Zijlstra, Bénédikt Tran, + and Pablo Galindo Salgado in :gh:`132661`.) + Improved modules ================ @@ -1226,8 +1063,6 @@ argparse and subparser names if mistyped by the user. (Contributed by Savannah Ostrowski in :gh:`124456`.) - .. _whatsnew314-color-argparse: - * Enable color for help text, which can be disabled with the optional *color* parameter to :class:`argparse.ArgumentParser`. This can also be controlled by :ref:`environment variables @@ -1238,7 +1073,7 @@ argparse ast --- -* Add :func:`ast.compare` for comparing two ASTs. +* Add :func:`~ast.compare`, a function for comparing two ASTs. (Contributed by Batuhan Taskaya and Jeremy Hylton in :gh:`60191`.) * Add support for :func:`copy.replace` for AST nodes. @@ -1247,15 +1082,17 @@ ast * Docstrings are now removed from an optimized AST in optimization level 2. (Contributed by Irit Katriel in :gh:`123958`.) -* The ``repr()`` output for AST nodes now includes more information. +* The :func:`repr` output for AST nodes now includes more information. (Contributed by Tomas Roun in :gh:`116022`.) -* :func:`ast.parse`, when called with an AST as input, now always verifies - that the root node type is appropriate. +* When called with an AST as input, the :func:`~ast.parse` function + now always verifies that the root node type is appropriate. (Contributed by Irit Katriel in :gh:`130139`.) -* Add new ``--feature-version``, ``--optimize``, ``--show-empty`` options to - command-line interface. +* Add new options to the command-line interface: + :option:`--feature-version `, + :option:`--optimize `, and + :option:`--show-empty `. (Contributed by Semyon Moroz in :gh:`133367`.) @@ -1274,21 +1111,23 @@ asyncio :meth:`asyncio.create_task`, :meth:`asyncio.loop.create_task`, :meth:`asyncio.TaskGroup.create_task`. - (Contributed by Thomas Grainger in :gh:`128307`.) - - -bdb ---- -* The :mod:`bdb` module now supports the :mod:`sys.monitoring` backend. - (Contributed by Tian Gao in :gh:`124533`.) + (Contributed by Thomas Grainger in :gh:`128307`.) +* There are two new utility functions for + introspecting and printing a program's call graph: + :func:`~asyncio.capture_call_graph` and :func:`~asyncio.print_call_graph`. + See :ref:`Asyncio introspection capabilities + ` for more details. + (Contributed by Yury Selivanov, Pablo Galindo Salgado, and Łukasz Langa + in :gh:`91048`.) - .. _whatsnew314-color-calendar: calendar -------- +.. _whatsnew314-color-calendar: + * By default, today's date is highlighted in color in :mod:`calendar`'s :ref:`command-line ` text output. This can be controlled by :ref:`environment variables @@ -1301,19 +1140,25 @@ concurrent.futures .. _whatsnew314-concurrent-futures-interp-pool: -* Add :class:`~concurrent.futures.InterpreterPoolExecutor`, - which exposes "subinterpreters" (multiple Python interpreters in the - same process) to Python code. This is separate from the proposed API - in :pep:`734`. +* Add a new executor class, :class:`~concurrent.futures.InterpreterPoolExecutor`, + which exposes multiple Python interpreters in the same process + ('subinterpreters') to Python code. + This uses a pool of independent Python interpreters to execute calls + asynchronously. + + This is separate from the new :mod:`~concurrent.interpreters` module + introduced by :ref:`PEP 734 `. (Contributed by Eric Snow in :gh:`124548`.) .. _whatsnew314-concurrent-futures-start-method: -* The default :class:`~concurrent.futures.ProcessPoolExecutor` - :ref:`start method ` changed - from :ref:`fork ` to :ref:`forkserver - ` on platforms other than macOS and - Windows where it was already :ref:`spawn `. +* On Unix platforms other than macOS, :ref:`'forkserver' + ` is now the default :ref:`start + method ` for + :class:`~concurrent.futures.ProcessPoolExecutor` + (replacing :ref:`'fork' `). + This change does not affect Windows or macOS, where :ref:`'spawn' + ` remains the default start method. If the threading incompatible *fork* method is required, you must explicitly request it by supplying a multiprocessing context *mp_context* to @@ -1326,31 +1171,36 @@ concurrent.futures (Contributed by Gregory P. Smith in :gh:`84559`.) -* Add :meth:`concurrent.futures.ProcessPoolExecutor.terminate_workers` and - :meth:`concurrent.futures.ProcessPoolExecutor.kill_workers` as - ways to terminate or kill all living worker processes in the given pool. +* Add two new methods to :class:`~concurrent.futures.ProcessPoolExecutor`, + :meth:`~concurrent.futures.ProcessPoolExecutor.terminate_workers` + and :meth:`~concurrent.futures.ProcessPoolExecutor.kill_workers`, + as ways to terminate or kill all living worker processes in the given pool. (Contributed by Charles Machalow in :gh:`130849`.) -* Add the optional ``buffersize`` parameter to - :meth:`concurrent.futures.Executor.map` to limit the number of submitted +* Add the optional *buffersize* parameter to :meth:`Executor.map + ` to limit the number of submitted tasks whose results have not yet been yielded. If the buffer is full, iteration over the *iterables* pauses until a result is yielded from the buffer. (Contributed by Enzo Bonnal and Josh Rosenberg in :gh:`74028`.) + configparser ------------ -* Security fix: will no longer write config files it cannot read. Attempting - to :meth:`configparser.ConfigParser.write` keys containing delimiters or - beginning with the section header pattern will raise a - :class:`configparser.InvalidWriteError`. +* :mod:`!configparser` will no longer write config files it cannot read, + to improve security. + Attempting to :meth:`~configparser.ConfigParser.write` keys containing + delimiters or beginning with the section header pattern will raise an + :class:`~configparser.InvalidWriteError`. (Contributed by Jacob Lincoln in :gh:`129270`.) + contextvars ----------- -* Support context manager protocol by :class:`contextvars.Token`. +* Support the :term:`context manager` protocol + for :class:`~contextvars.Token` objects. (Contributed by Andrew Svetlov in :gh:`129889`.) @@ -1358,8 +1208,8 @@ ctypes ------ * The layout of :ref:`bit fields ` - in :class:`~ctypes.Structure` and :class:`~ctypes.Union` - now matches platform defaults (GCC/Clang or MSVC) more closely. + in :class:`~ctypes.Structure` and :class:`~ctypes.Union` objects + is now a closer match to platform defaults (GCC/Clang or MSVC). In particular, fields no longer overlap. (Contributed by Matthias Görgens in :gh:`97702`.) @@ -1378,7 +1228,7 @@ ctypes * On Windows, the :func:`~ctypes.CopyComPointer` function is now public. (Contributed by Jun Komoda in :gh:`127275`.) -* :func:`ctypes.memoryview_at` now exists to create a +* Add :func:`~ctypes.memoryview_at`, a function to create a :class:`memoryview` object that refers to the supplied pointer and length. This works like :func:`ctypes.string_at` except it avoids a buffer copy, and is typically useful when implementing pure Python @@ -1386,7 +1236,7 @@ ctypes (Contributed by Rian Hunter in :gh:`112018`.) * Complex types, :class:`~ctypes.c_float_complex`, - :class:`~ctypes.c_double_complex` and :class:`~ctypes.c_longdouble_complex`, + :class:`~ctypes.c_double_complex`, and :class:`~ctypes.c_longdouble_complex`, are now available if both the compiler and the ``libffi`` library support complex C types. (Contributed by Sergey B Kirpichev in :gh:`61103`.) @@ -1396,50 +1246,57 @@ ctypes (Contributed by Brian Ward in :gh:`119349`.) * Move :func:`ctypes.POINTER` types cache from a global internal cache - (``_pointer_type_cache``) to the :attr:`ctypes._CData.__pointer_type__` - attribute of the corresponding :mod:`ctypes` types. + (``_pointer_type_cache``) to the :attr:`_CData.__pointer_type__ + ` attribute of the corresponding + :mod:`!ctypes` types. This will stop the cache from growing without limits in some situations. (Contributed by Sergey Miryanov in :gh:`100926`.) -* The :class:`ctypes.py_object` type now supports subscription, +* The :class:`~ctypes.py_object` type now supports subscription, making it a :term:`generic type`. (Contributed by Brian Schubert in :gh:`132168`.) -* :mod:`ctypes` now supports :term:`free-threading builds `. +* :mod:`!ctypes` now supports :term:`free-threading builds `. (Contributed by Kumar Aditya and Peter Bierma in :gh:`127945`.) + curses ------ * Add the :func:`~curses.assume_default_colors` function, a refinement of the :func:`~curses.use_default_colors` function which - allows to change the color pair ``0``. + allows changing the color pair ``0``. (Contributed by Serhiy Storchaka in :gh:`133139`.) + datetime -------- -* Add :meth:`datetime.time.strptime` and :meth:`datetime.date.strptime`. +* Add the :meth:`~datetime.date.strptime` method to the + :class:`datetime.date` and :class:`datetime.time` classes. (Contributed by Wannes Boeykens in :gh:`41431`.) + decimal ------- -* Add alternative :class:`~decimal.Decimal` constructor - :meth:`Decimal.from_number() `. +* Add :meth:`.Decimal.from_number` as an alternative constructor for + :class:`~decimal.Decimal`. (Contributed by Serhiy Storchaka in :gh:`121798`.) -* Expose :func:`decimal.IEEEContext` to support creation of contexts +* Expose :func:`~decimal.IEEEContext` to support creation of contexts corresponding to the IEEE 754 (2008) decimal interchange formats. (Contributed by Sergey B Kirpichev in :gh:`53032`.) + difflib ------- * Comparison pages with highlighted changes generated by the - :class:`difflib.HtmlDiff` class now support dark mode. + :class:`~difflib.HtmlDiff` class now support 'dark mode'. (Contributed by Jiahao Li in :gh:`129939`.) + dis --- @@ -1464,7 +1321,7 @@ dis errno ----- -* Add :data:`errno.EHWPOISON` error code. +* Add the :data:`~errno.EHWPOISON` error code constant. (Contributed by James Roy in :gh:`126585`.) @@ -1472,73 +1329,47 @@ faulthandler ------------ * Add support for printing the C stack trace on systems that - :ref:`support it ` via :func:`faulthandler.dump_c_stack` - or via the *c_stack* argument in :func:`faulthandler.enable`. + :ref:`support it ` via the new + :func:`~faulthandler.dump_c_stack` function or via the *c_stack* argument + in :func:`faulthandler.enable`. (Contributed by Peter Bierma in :gh:`127604`.) fnmatch ------- -* Added :func:`fnmatch.filterfalse` for excluding names matching a pattern. +* Add :func:`~fnmatch.filterfalse`, a function to reject names + matching a given pattern. (Contributed by Bénédikt Tran in :gh:`74598`.) fractions --------- -* Add support for converting any objects that have the - :meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. +* A :class:`~fractions.Fraction` object may now be constructed from any + object with the :meth:`!as_integer_ratio` method. (Contributed by Serhiy Storchaka in :gh:`82017`.) -* Add alternative :class:`~fractions.Fraction` constructor - :meth:`Fraction.from_number() `. +* Add :meth:`.Fraction.from_number` as an alternative constructor for + :class:`~fractions.Fraction`. (Contributed by Serhiy Storchaka in :gh:`121797`.) functools --------- -* Add support to :func:`functools.partial` and - :func:`functools.partialmethod` for :data:`functools.Placeholder` sentinels - to reserve a place for positional arguments. +* Add the :data:`~functools.Placeholder` sentinel. + This may be used with the :func:`~functools.partial` + or :func:`~functools.partialmethod` functions to reserve a place + for positional arguments in the returned :ref:`partial object + `. (Contributed by Dominykas Grigonis in :gh:`119127`.) -* Allow the *initial* parameter of :func:`functools.reduce` to be passed +* Allow the *initial* parameter of :func:`~functools.reduce` to be passed as a keyword argument. (Contributed by Sayandip Dutta in :gh:`125916`.) -gc --- - -The cyclic garbage collector is now incremental, -which changes the meaning of the results of -:meth:`~gc.get_threshold` and :meth:`~gc.set_threshold` -as well as :meth:`~gc.get_count` and :meth:`~gc.get_stats`. - -* For backwards compatibility, :meth:`~gc.get_threshold` continues to return - a three-item tuple. - The first value is the threshold for young collections, as before; - the second value determines the rate at which the old collection is scanned - (the default is 10, and higher values mean that the old collection - is scanned more slowly). - The third value is meaningless and is always zero. - -* :meth:`~gc.set_threshold` ignores any items after the second. - -* :meth:`~gc.get_count` and :meth:`~gc.get_stats` continue to return - the same format of results. - The only difference is that instead of the results referring to - the young, aging and old generations, - the results refer to the young generation - and the aging and collecting spaces of the old generation. - -In summary, code that attempted to manipulate the behavior of the cycle GC -may not work exactly as intended, but it is very unlikely to be harmful. -All other code will work just fine. - - getopt ------ @@ -1552,16 +1383,17 @@ getopt getpass ------- -* Support keyboard feedback by :func:`getpass.getpass` via the keyword-only - optional argument ``echo_char``. Placeholder characters are rendered whenever - a character is entered, and removed when a character is deleted. +* Support keyboard feedback in the :func:`~getpass.getpass` function via + the keyword-only optional argument *echo_char*. + Placeholder characters are rendered whenever a character is entered, + and removed when a character is deleted. (Contributed by Semyon Moroz in :gh:`77065`.) graphlib -------- -* Allow :meth:`graphlib.TopologicalSorter.prepare` to be called more than once +* Allow :meth:`.TopologicalSorter.prepare` to be called more than once as long as sorting has not started. (Contributed by Daniel Pope in :gh:`130914`.) @@ -1569,13 +1401,14 @@ graphlib heapq ----- -* Add functions for working with max-heaps: +* The :mod:`!heapq` module has improved support for working with max-heaps, + via the following new functions: - * :func:`heapq.heapify_max`, - * :func:`heapq.heappush_max`, - * :func:`heapq.heappop_max`, - * :func:`heapq.heapreplace_max` - * :func:`heapq.heappushpop_max` + * :func:`~heapq.heapify_max` + * :func:`~heapq.heappush_max` + * :func:`~heapq.heappop_max` + * :func:`~heapq.heapreplace_max` + * :func:`~heapq.heappushpop_max` hmac @@ -1583,6 +1416,8 @@ hmac * Add a built-in implementation for HMAC (:rfc:`2104`) using formally verified code from the `HACL* `__ project. + This implementation is used as a fallback when the OpenSSL implementation + of HMAC is not available. (Contributed by Bénédikt Tran in :gh:`99108`.) @@ -1598,9 +1433,12 @@ http the command-line interface (``python -m http.server``) through the following options: - * ``--tls-cert ``: Path to the TLS certificate file. - * ``--tls-key ``: Optional path to the private key file. - * ``--tls-password-file ``: Optional path to the password file for the private key. + * :option:`--tls-cert \ `: + Path to the TLS certificate file. + * :option:`--tls-key \ `: + Optional path to the private key file. + * :option:`--tls-password-file \ `: + Optional path to the password file for the private key. (Contributed by Semyon Moroz in :gh:`85162`.) @@ -1608,7 +1446,7 @@ http imaplib ------- -* Add :meth:`IMAP4.idle() `, implementing the IMAP4 +* Add :meth:`.IMAP4.idle`, implementing the IMAP4 ``IDLE`` command as defined in :rfc:`2177`. (Contributed by Forest in :gh:`55454`.) @@ -1616,15 +1454,16 @@ imaplib inspect ------- -* :func:`inspect.signature` takes a new argument *annotation_format* to control +* :func:`~inspect.signature` takes a new argument *annotation_format* to control the :class:`annotationlib.Format` used for representing annotations. (Contributed by Jelle Zijlstra in :gh:`101552`.) -* :meth:`inspect.Signature.format` takes a new argument *unquote_annotations*. - If true, string :term:`annotations ` are displayed without surrounding quotes. +* :meth:`.Signature.format` takes a new argument *unquote_annotations*. + If true, string :term:`annotations ` are displayed without + surrounding quotes. (Contributed by Jelle Zijlstra in :gh:`101552`.) -* Add function :func:`inspect.ispackage` to determine whether an object is a +* Add function :func:`~inspect.ispackage` to determine whether an object is a :term:`package` or not. (Contributed by Zhikang Yan in :gh:`125634`.) @@ -1636,7 +1475,7 @@ io :exc:`BlockingIOError` if the operation cannot immediately return bytes. (Contributed by Giovanni Siragusa in :gh:`109523`.) -* Add protocols :class:`io.Reader` and :class:`io.Writer` as a simpler +* Add the :class:`~io.Reader` and :class:`~io.Writer` protocols as simpler alternatives to the pseudo-protocols :class:`typing.IO`, :class:`typing.TextIO`, and :class:`typing.BinaryIO`. (Contributed by Sebastian Rittau in :gh:`127648`.) @@ -1645,35 +1484,36 @@ io json ---- -* Add notes for JSON serialization errors that allow to identify the source - of the error. +* Add exception notes for JSON serialization errors that allow + identifying the source of the error. (Contributed by Serhiy Storchaka in :gh:`122163`.) -* Enable the :mod:`json` module to work as a script using the :option:`-m` - switch: :program:`python -m json`. +* Allow using the :mod:`json` module as a script using the :option:`-m` switch: + :program:`python -m json`. + This is now preferred to :program:`python -m json.tool`, + which is :term:`soft deprecated`. See the :ref:`JSON command-line interface ` documentation. (Contributed by Trey Hunner in :gh:`122873`.) - .. _whatsnew314-color-json: - * By default, the output of the :ref:`JSON command-line interface ` is highlighted in color. This can be controlled by :ref:`environment variables `. (Contributed by Tomas Roun in :gh:`131952`.) + linecache --------- -* :func:`linecache.getline` can retrieve source code for frozen modules. +* :func:`~linecache.getline` can now retrieve source code for frozen modules. (Contributed by Tian Gao in :gh:`131638`.) logging.handlers ---------------- -* :class:`logging.handlers.QueueListener` now implements the context - manager protocol, allowing it to be used in a :keyword:`with` statement. +* :class:`~logging.handlers.QueueListener` objects now support the + :term:`context manager` protocol. (Contributed by Charles Machalow in :gh:`132106`.) * :meth:`QueueListener.start ` now @@ -1691,14 +1531,13 @@ math mimetypes --------- -* Document the command-line for :mod:`mimetypes`. - It now exits with ``1`` on failure instead of ``0`` - and ``2`` on incorrect command-line parameters instead of ``1``. - Also, errors are printed to stderr instead of stdout and their text is made - tighter. +* Add a public :ref:`command-line ` for the module, + invoked via :program:`python -m mimetypes`. (Contributed by Oleg Iarygin and Hugo van Kemenade in :gh:`93096`.) -* Add MS and :rfc:`8081` MIME types for fonts: +* Add several new MIME types based on RFCs and common usage: + + .. rubric:: Microsoft and :rfc:`8081` MIME types for fonts * Embedded OpenType: ``application/vnd.ms-fontobject`` * OpenType Layout (OTF) ``font/otf`` @@ -1706,18 +1545,14 @@ mimetypes * WOFF 1.0 ``font/woff`` * WOFF 2.0 ``font/woff2`` - (Contributed by Sahil Prajapati and Hugo van Kemenade in :gh:`84852`.) - -* Add :rfc:`9559` MIME types for Matroska audiovisual data container - structures, containing: + .. rubric:: :rfc:`9559` MIME types for Matroska audiovisual + data container structures * audio with no video: ``audio/matroska`` (``.mka``) * video: ``video/matroska`` (``.mkv``) * stereoscopic video: ``video/matroska-3d`` (``.mk3d``) - (Contributed by Hugo van Kemenade in :gh:`89416`.) - -* Add MIME types for images with RFCs: + .. rubric:: Images with RFCs * :rfc:`1494`: CCITT Group 3 (``.g3``) * :rfc:`3362`: Real-time Facsimile, T.38 (``.t38``) @@ -1726,9 +1561,7 @@ mimetypes * :rfc:`4047`: Flexible Image Transport System (``.fits``) * :rfc:`7903`: Enhanced Metafile (``.emf``) and Windows Metafile (``.wmf``) - (Contributed by Hugo van Kemenade in :gh:`85957`.) - -* More MIME type changes: + .. rubric:: Other MIME type additions and changes * :rfc:`2361`: Change type for ``.avi`` to ``video/vnd.avi`` and for ``.wav`` to ``audio/vnd.wave`` @@ -1736,6 +1569,8 @@ mimetypes * :rfc:`5334`: Add Ogg media (``.oga``, ``.ogg`` and ``.ogx``) * :rfc:`6713`: Add gzip ``application/gzip`` (``.gz``) * :rfc:`9639`: Add FLAC ``audio/flac`` (``.flac``) + * :rfc:`9512` ``application/yaml`` MIME type for YAML files (``.yaml`` + and ``.yml``) * Add 7z ``application/x-7z-compressed`` (``.7z``) * Add Android Package ``application/vnd.android.package-archive`` (``.apk``) when not strict @@ -1758,11 +1593,9 @@ mimetypes * `W3C `__: Add EPUB ``application/epub+zip`` (``.epub``) - (Contributed by Hugo van Kemenade in :gh:`129965`.) - -* Add :rfc:`9512` ``application/yaml`` MIME type for YAML files (``.yaml`` - and ``.yml``). (Contributed by Sasha "Nelie" Chernykh and Hugo van Kemenade - in :gh:`132056`.) + (Contributed by Sahil Prajapati and Hugo van Kemenade in :gh:`84852`, + by Sasha "Nelie" Chernykh and Hugo van Kemenade in :gh:`132056`, + and by Hugo van Kemenade in :gh:`89416`, :gh:`85957`, and :gh:`129965`.) multiprocessing @@ -1770,14 +1603,16 @@ multiprocessing .. _whatsnew314-multiprocessing-start-method: -* The default :ref:`start method ` changed - from :ref:`fork ` to :ref:`forkserver - ` on platforms other than macOS and - Windows where it was already :ref:`spawn `. +* On Unix platforms other than macOS, :ref:`'forkserver' + ` is now the default :ref:`start + method ` + (replacing :ref:`'fork' `). + This change does not affect Windows or macOS, where :ref:`'spawn' + ` remains the default start method. If the threading incompatible *fork* method is required, you must explicitly - request it via a context from :func:`multiprocessing.get_context` (preferred) - or change the default via :func:`multiprocessing.set_start_method`. + request it via a context from :func:`~multiprocessing.get_context` (preferred) + or change the default via :func:`~multiprocessing.set_start_method`. See :ref:`forkserver restrictions ` for information and differences with the *fork* method and how this change @@ -1786,7 +1621,7 @@ multiprocessing (Contributed by Gregory P. Smith in :gh:`84559`.) -* :mod:`multiprocessing`'s ``"forkserver"`` start method now authenticates +* :mod:`multiprocessing`'s ``'forkserver'`` start method now authenticates its control socket to avoid solely relying on filesystem permissions to restrict what other processes could cause the forkserver to spawn workers and run code. @@ -1802,20 +1637,22 @@ multiprocessing (Contributed by Roy Hyunjin Han for :gh:`103134`.) * Add support for shared :class:`set` objects via - :meth:`SyncManager.set() `. - The :func:`set` in :func:`multiprocessing.Manager` method is now available. + :meth:`.SyncManager.set`. + The :func:`set` in :func:`~multiprocessing.Manager` method is now available. (Contributed by Mingyu Park in :gh:`129949`.) -* Add :func:`multiprocessing.Process.interrupt` which terminates the child +* Add the :meth:`~multiprocessing.Process.interrupt` + to :class:`multiprocessing.Process` objects, which terminates the child process by sending :py:const:`~signal.SIGINT`. This enables :keyword:`finally` clauses to print a stack trace for the terminated process. (Contributed by Artem Pulkin in :gh:`131913`.) + operator -------- -* Two new functions :func:`operator.is_none` and :func:`operator.is_not_none` - have been added, such that ``operator.is_none(obj)`` is equivalent +* Add :func:`~operator.is_none` and :func:`~operator.is_not_none` as a pair + of functions, such that ``operator.is_none(obj)`` is equivalent to ``obj is None`` and ``operator.is_not_none(obj)`` is equivalent to ``obj is not None``. (Contributed by Raymond Hettinger and Nico Mexis in :gh:`115808`.) @@ -1824,17 +1661,17 @@ operator os -- -* Add the :func:`os.reload_environ` function to update :data:`os.environ` and +* Add the :func:`~os.reload_environ` function to update :data:`os.environ` and :data:`os.environb` with changes to the environment made by :func:`os.putenv`, by :func:`os.unsetenv`, or made outside Python in the same process. (Contributed by Victor Stinner in :gh:`120057`.) * Add the :data:`~os.SCHED_DEADLINE` and :data:`~os.SCHED_NORMAL` constants - to the :mod:`os` module. + to the :mod:`!os` module. (Contributed by James Roy in :gh:`127688`.) -* Add the :func:`os.readinto` function to read into a +* Add the :func:`~os.readinto` function to read into a :ref:`buffer object ` from a file descriptor. (Contributed by Cody Maloney in :gh:`129205`.) @@ -1842,8 +1679,8 @@ os os.path ------- -* The *strict* parameter to :func:`os.path.realpath` accepts a new value, - :data:`os.path.ALLOW_MISSING`. +* The *strict* parameter to :func:`~os.path.realpath` accepts a new value, + :data:`~os.path.ALLOW_MISSING`. If used, errors other than :exc:`FileNotFoundError` will be re-raised; the resulting path can be missing but it will be free of symlinks. (Contributed by Petr Viktorin for :cve:`2025-4517`.) @@ -1862,8 +1699,8 @@ pathlib (Contributed by Barney Gale in :gh:`73991`.) -* Add :attr:`pathlib.Path.info` attribute, which stores an object - implementing the :class:`pathlib.types.PathInfo` protocol (also new). The +* Add the :attr:`~pathlib.Path.info` attribute, which stores an object + implementing the new :class:`pathlib.types.PathInfo` protocol. The object supports querying the file type and internally caching :func:`~os.stat` results. Path objects generated by :meth:`~pathlib.Path.iterdir` are initialized with file type information @@ -1874,7 +1711,26 @@ pathlib pdb --- -* Hardcoded breakpoints (:func:`breakpoint` and :func:`pdb.set_trace`) now +* The :mod:`pdb` module now supports remote attaching to a running Python process + using a new :option:`-p PID ` command-line option: + + .. code-block:: sh + + python -m pdb -p 1234 + + This will connect to the Python process with the given PID and allow you to + debug it interactively. Notice that due to how the Python interpreter works + attaching to a remote process that is blocked in a system call or waiting for + I/O will only work once the next bytecode instruction is executed or when the + process receives a signal. + + This feature uses :ref:`PEP 768 ` + and the new :func:`sys.remote_exec` function to attach to the remote process + and send the PDB commands to it. + + (Contributed by Matt Wozniski and Pablo Galindo in :gh:`131591`.) + +* Hardcoded breakpoints (:func:`breakpoint` and :func:`~pdb.set_trace`) now reuse the most recent :class:`~pdb.Pdb` instance that calls :meth:`~pdb.Pdb.set_trace`, instead of creating a new one each time. As a result, all the instance specific data like :pdbcmd:`display` and @@ -1907,21 +1763,14 @@ pdb * ``$_asynctask`` is added to access the current asyncio task if applicable. (Contributed by Tian Gao in :gh:`124367`.) -* :mod:`pdb` now supports two backends: :func:`sys.settrace` and - :mod:`sys.monitoring`. Using :mod:`pdb` CLI or :func:`breakpoint` will - always use the :mod:`sys.monitoring` backend. Explicitly instantiating - :class:`pdb.Pdb` and its derived classes will use the :func:`sys.settrace` - backend by default, which is configurable. - (Contributed by Tian Gao in :gh:`124533`.) - * :func:`pdb.set_trace_async` is added to support debugging asyncio coroutines. :keyword:`await` statements are supported with this function. (Contributed by Tian Gao in :gh:`132576`.) * Source code displayed in :mod:`pdb` will be syntax-highlighted. This feature - can be controlled using the same methods as PyREPL, in addition to the newly - added ``colorize`` argument of :class:`pdb.Pdb`. + can be controlled using the same methods as the default :term:`interactive` + shell, in addition to the newly added ``colorize`` argument of :class:`pdb.Pdb`. (Contributed by Tian Gao and Łukasz Langa in :gh:`133355`.) @@ -1931,15 +1780,16 @@ pickle * Set the default protocol version on the :mod:`pickle` module to 5. For more details, see :ref:`pickle protocols `. -* Add notes for pickle serialization errors that allow to identify the source - of the error. +* Add exception notes for pickle serialization errors that allow + identifying the source of the error. (Contributed by Serhiy Storchaka in :gh:`122213`.) platform -------- -* Add :func:`platform.invalidate_caches` to invalidate the cached results. +* Add :func:`~platform.invalidate_caches`, a function to invalidate + cached results in the :mod:`!platform` module. (Contributed by Bénédikt Tran in :gh:`122549`.) @@ -1951,6 +1801,19 @@ pydoc (Contributed by Jelle Zijlstra in :gh:`101552`.) +re +-- + +* Support ``\z`` as a synonym for ``\Z`` in :mod:`regular expressions `. + It is interpreted unambiguously in many other regular expression engines, + unlike ``\Z``, which has subtly different behavior. + (Contributed by Serhiy Storchaka in :gh:`133306`.) + +* ``\B`` in :mod:`regular expression ` now matches the empty input string, + meaning that it is now always the opposite of ``\b``. + (Contributed by Serhiy Storchaka in :gh:`124130`.) + + socket ------ @@ -1977,11 +1840,12 @@ socket * Add many new constants. (Contributed by Serhiy Storchaka in :gh:`132734`.) + ssl --- -* Indicate through :data:`ssl.HAS_PHA` whether the :mod:`ssl` module supports - TLSv1.3 post-handshake client authentication (PHA). +* Indicate through the :data:`~ssl.HAS_PHA` Boolean whether the :mod:`!ssl` + module supports TLSv1.3 post-handshake client authentication (PHA). (Contributed by Will Childs-Klein in :gh:`128036`.) @@ -1997,7 +1861,7 @@ struct symtable -------- -* Expose the following :class:`symtable.Symbol` methods: +* Expose the following :class:`~symtable.Symbol` methods: * :meth:`~symtable.Symbol.is_comp_cell` * :meth:`~symtable.Symbol.is_comp_iter` @@ -2012,28 +1876,41 @@ sys * The previously undocumented special function :func:`sys.getobjects`, which only exists in specialized builds of Python, may now return objects from other interpreters than the one it's called in. + (Contributed by Eric Snow in :gh:`125286`.) * Add :func:`sys._is_immortal` for determining if an object is :term:`immortal`. (Contributed by Peter Bierma in :gh:`128509`.) -* On FreeBSD, :data:`sys.platform` doesn't contain the major version anymore. +* On FreeBSD, :data:`sys.platform` no longer contains the major version number. It is always ``'freebsd'``, instead of ``'freebsd13'`` or ``'freebsd14'``. + (Contributed by Michael Osipov in :gh:`129393`.) * Raise :exc:`DeprecationWarning` for :func:`sys._clear_type_cache`. This function was deprecated in Python 3.13 but it didn't raise a runtime warning. +* Add :func:`sys.remote_exec` to implement the new external debugger interface. + See :ref:`PEP 768 ` for details. + (Contributed by Pablo Galindo Salgado, Matt Wozniski, and Ivona Stojanovic + in :gh:`131591`.) + +* Add the :data:`sys._jit` namespace, containing utilities for introspecting + just-in-time compilation. + (Contributed by Brandt Bucher in :gh:`133231`.) + sys.monitoring -------------- -* Two new events are added: :monitoring-event:`BRANCH_LEFT` and - :monitoring-event:`BRANCH_RIGHT`. The ``BRANCH`` event is deprecated. +* Add two new monitoring events, :monitoring-event:`BRANCH_LEFT` and + :monitoring-event:`BRANCH_RIGHT`. + These replace and deprecate the :monitoring-event:`!BRANCH` event. + (Contributed by Mark Shannon in :gh:`122548`.) sysconfig --------- -* Add ``ABIFLAGS`` key to :func:`sysconfig.get_config_vars` on Windows. +* Add ``ABIFLAGS`` key to :func:`~sysconfig.get_config_vars` on Windows. (Contributed by Xuehai Pan in :gh:`131799`.) @@ -2043,15 +1920,18 @@ tarfile * :func:`~tarfile.data_filter` now normalizes symbolic link targets in order to avoid path traversal attacks. (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2025-4138`.) + * :func:`~tarfile.TarFile.extractall` now skips fixing up directory attributes when a directory was removed or replaced by another kind of file. (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2024-12718`.) + * :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` now (re-)apply the extraction filter when substituting a link (hard or symbolic) with a copy of another archive member, and when fixing up directory attributes. The former raises a new exception, :exc:`~tarfile.LinkFallbackError`. (Contributed by Petr Viktorin for :cve:`2025-4330` and :cve:`2024-12718`.) + * :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` no longer extract rejected members when :func:`~tarfile.TarFile.errorlevel` is zero. @@ -2071,17 +1951,18 @@ tkinter ------- * Make :mod:`tkinter` widget methods :meth:`!after` and :meth:`!after_idle` - accept arguments passed by keyword. + accept keyword arguments. (Contributed by Zhikang Yan in :gh:`126899`.) -* Add ability to specify name for :class:`!tkinter.OptionMenu` and +* Add ability to specify a name for :class:`!tkinter.OptionMenu` and :class:`!tkinter.ttk.OptionMenu`. (Contributed by Zhikang Yan in :gh:`130482`.) + turtle ------ -* Add context managers for :func:`turtle.fill`, :func:`turtle.poly` +* Add context managers for :func:`turtle.fill`, :func:`turtle.poly`, and :func:`turtle.no_animation`. (Contributed by Marie Roald and Yngve Mardal Moe in :gh:`126350`.) @@ -2099,43 +1980,60 @@ typing .. _whatsnew314-typing-union: -* :class:`types.UnionType` and :class:`typing.Union` are now aliases for each other, - meaning that both old-style unions (created with ``Union[int, str]``) and new-style - unions (``int | str``) now create instances of the same runtime type. This unifies - the behavior between the two syntaxes, but leads to some differences in behavior that +* The :class:`types.UnionType` and :class:`typing.Union` types are now + aliases for each other, meaning that both old-style unions + (created with ``Union[int, str]``) and new-style unions (``int | str``) + now create instances of the same runtime type. This unifies the behavior + between the two syntaxes, but leads to some differences in behavior that may affect users who introspect types at runtime: - - Both syntaxes for creating a union now produce the same string representation in - ``repr()``. For example, ``repr(Union[int, str])`` - is now ``"int | str"`` instead of ``"typing.Union[int, str]"``. - - Unions created using the old syntax are no longer cached. Previously, running - ``Union[int, str]`` multiple times would return the same object - (``Union[int, str] is Union[int, str]`` would be ``True``), but now it will - return two different objects. Users should use ``==`` to compare unions for equality, not - ``is``. New-style unions have never been cached this way. - This change could increase memory usage for some programs that use a large number of - unions created by subscripting ``typing.Union``. However, several factors offset this cost: - unions used in annotations are no longer evaluated by default in Python 3.14 - because of :pep:`649`; an instance of :class:`types.UnionType` is - itself much smaller than the object returned by ``Union[]`` was on prior Python - versions; and removing the cache also saves some space. It is therefore - unlikely that this change will cause a significant increase in memory usage for most - users. + - Both syntaxes for creating a union now produce the same string + representation in :func:`repr`. + For example, ``repr(Union[int, str])`` is now ``"int | str"`` instead of + ``"typing.Union[int, str]"``. + + - Unions created using the old syntax are no longer cached. + Previously, running ``Union[int, str]`` multiple times would return + the same object (``Union[int, str] is Union[int, str]`` would be ``True``), + but now it will return two different objects. + Use ``==`` to compare unions for equality, not ``is``. + New-style unions have never been cached this way. + This change could increase memory usage for some programs that use + a large number of unions created by subscripting ``typing.Union``. + However, several factors offset this cost: + unions used in annotations are no longer evaluated by default in Python + 3.14 because of :pep:`649`; an instance of :class:`types.UnionType` is + itself much smaller than the object returned by ``Union[]`` was on prior + Python versions; and removing the cache also saves some space. + It is therefore unlikely that this change will cause a significant increase + in memory usage for most users. + - Previously, old-style unions were implemented using the private class - ``typing._UnionGenericAlias``. This class is no longer needed for the implementation, - but it has been retained for backward compatibility, with removal scheduled for Python - 3.17. Users should use documented introspection helpers like :func:`typing.get_origin` - and :func:`typing.get_args` instead of relying on private implementation details. - - It is now possible to use :class:`typing.Union` itself in :func:`isinstance` checks. - For example, ``isinstance(int | str, typing.Union)`` will return ``True``; previously - this raised :exc:`TypeError`. - - The ``__args__`` attribute of :class:`typing.Union` objects is no longer writable. - - It is no longer possible to set any attributes on :class:`typing.Union` objects. + ``typing._UnionGenericAlias``. + This class is no longer needed for the implementation, + but it has been retained for backward compatibility, + with removal scheduled for Python 3.17. + Users should use documented introspection helpers like + :func:`~typing.get_origin` and :func:`typing.get_args` instead of + relying on private implementation details. + + - It is now possible to use :class:`typing.Union` itself in + :func:`isinstance` checks. + For example, ``isinstance(int | str, typing.Union)`` will return ``True``; + previously this raised :exc:`TypeError`. + + - The :attr:`!__args__` attribute of :class:`typing.Union` objects is + no longer writable. + + - It is no longer possible to set any attributes on :class:`~typing.Union` + objects. This only ever worked for dunder attributes on previous versions, was never documented to work, and was subtly broken in many cases. (Contributed by Jelle Zijlstra in :gh:`105499`.) +* :class:`~typing.TypeAliasType` now supports star unpacking. + unicodedata ----------- @@ -2143,11 +2041,11 @@ unicodedata * The Unicode database has been updated to Unicode 16.0.0. -.. _whatsnew314-color-unittest: - unittest -------- +.. _whatsnew314-color-unittest: + * :mod:`unittest` output is now colored by default. This can be controlled by :ref:`environment variables `. @@ -2170,7 +2068,7 @@ unittest :meth:`~unittest.TestCase.assertNotStartsWith`, :meth:`~unittest.TestCase.assertEndsWith` and :meth:`~unittest.TestCase.assertNotEndsWith` check whether the Unicode - or byte string starts or ends with particular string(s). + or byte string starts or ends with particular strings. (Contributed by Serhiy Storchaka in :gh:`71339`.) @@ -2185,7 +2083,7 @@ urllib * Improve ergonomics and standards compliance when parsing and emitting ``file:`` URLs. - In :func:`urllib.request.url2pathname`: + In :func:`~urllib.request.url2pathname`: - Accept a complete URL when the new *require_scheme* argument is set to true. @@ -2196,7 +2094,7 @@ urllib - Raise :exc:`~urllib.error.URLError` if a URL authority isn't local, except on Windows where we return a UNC path as before. - In :func:`urllib.request.pathname2url`: + In :func:`~urllib.request.pathname2url`: - Return a complete URL when the new *add_scheme* argument is set to true. - Include an empty URL authority when a path begins with a slash. For @@ -2212,16 +2110,17 @@ urllib uuid ---- -* Add support for UUID versions 6, 7, and 8 via :func:`uuid.uuid6`, - :func:`uuid.uuid7`, and :func:`uuid.uuid8` respectively, as specified +* Add support for UUID versions 6, 7, and 8 via :func:`~uuid.uuid6`, + :func:`~uuid.uuid7`, and :func:`~uuid.uuid8` respectively, as specified in :rfc:`9562`. (Contributed by Bénédikt Tran in :gh:`89083`.) -* :const:`uuid.NIL` and :const:`uuid.MAX` are now available to represent the +* :const:`~uuid.NIL` and :const:`~uuid.MAX` are now available to represent the Nil and Max UUID formats as defined by :rfc:`9562`. (Contributed by Nick Pope in :gh:`128427`.) -* Allow to generate multiple UUIDs at once via :option:`python -m uuid --count `. +* Allow generating multiple UUIDs simultaneously on the command-line via + :option:`python -m uuid --count `. (Contributed by Simon Legner in :gh:`131236`.) @@ -2236,249 +2135,197 @@ webbrowser supported browsers on macOS. -zipinfo +zipfile ------- -* Added :func:`ZipInfo._for_archive ` +* Added :meth:`ZipInfo._for_archive `, a method to resolve suitable defaults for a :class:`~zipfile.ZipInfo` object as used by :func:`ZipFile.writestr `. (Contributed by Bénédikt Tran in :gh:`123424`.) -* :meth:`zipfile.ZipFile.writestr` now respect ``SOURCE_DATE_EPOCH`` that - distributions can set centrally and have build tools consume this in order - to produce reproducible output. +* :meth:`.ZipFile.writestr` now respects the :envvar:`SOURCE_DATE_EPOCH` + environment variable in order to better support reproducible builds. (Contributed by Jiahao Li in :gh:`91279`.) .. Add improved modules above alphabetically, not here at the end. + Optimizations ============= * The import time for several standard library modules has been improved, - including :mod:`ast`, :mod:`asyncio`, :mod:`base64`, :mod:`cmd`, :mod:`csv`, - :mod:`gettext`, :mod:`importlib.util`, :mod:`locale`, :mod:`mimetypes`, - :mod:`optparse`, :mod:`pickle`, :mod:`pprint`, :mod:`pstats`, :mod:`socket`, - :mod:`subprocess`, :mod:`threading`, :mod:`tomllib`, and :mod:`zipfile`. + including :mod:`annotationlib`, :mod:`ast`, :mod:`asyncio`, :mod:`base64`, + :mod:`cmd`, :mod:`csv`, :mod:`gettext`, :mod:`importlib.util`, :mod:`locale`, + :mod:`mimetypes`, :mod:`optparse`, :mod:`pickle`, :mod:`pprint`, + :mod:`pstats`, :mod:`shlex`, :mod:`socket`, :mod:`string`, :mod:`subprocess`, + :mod:`threading`, :mod:`tomllib`, :mod:`types`, and :mod:`zipfile`. (Contributed by Adam Turner, Bénédikt Tran, Chris Markiewicz, Eli Schwartz, Hugo van Kemenade, Jelle Zijlstra, and others in :gh:`118761`.) +* The interpreter now avoids some reference count modifications internally + when it's safe to do so. + This can lead to different values being returned from :func:`sys.getrefcount` + and :c:func:`Py_REFCNT` compared to previous versions of Python. + See :ref:`below ` for details. + asyncio ------- -* :mod:`asyncio` has a new per-thread double linked list implementation internally for - :class:`native tasks ` which speeds up execution by 10-20% on standard - pyperformance benchmarks and reduces memory usage. +* Standard benchmark results have improved by 10-20% following the + implementation of a new per-thread doubly linked list + for :class:`native tasks `, + also reducing memory usage. This enables external introspection tools such as :ref:`python -m asyncio pstree ` to introspect the call graph of asyncio tasks running in all threads. (Contributed by Kumar Aditya in :gh:`107803`.) -* :mod:`asyncio` has first class support for :term:`free-threading builds `. - This enables parallel execution of multiple event loops across different threads and scales - linearly with the number of threads. +* The module now has first class support for + :term:`free-threading builds `. + This enables parallel execution of multiple event loops across + different threads, scaling linearly with the number of threads. (Contributed by Kumar Aditya in :gh:`128002`.) -* :mod:`asyncio` has new utility functions for introspecting and printing - the program's call graph: :func:`asyncio.capture_call_graph` and - :func:`asyncio.print_call_graph`. - (Contributed by Yury Selivanov, Pablo Galindo Salgado, and Łukasz Langa - in :gh:`91048`.) - base64 ------ -* Improve the performance of :func:`base64.b16decode` by up to ten times, - and reduce the import time of :mod:`base64` by up to six times. - (Contributed by Bénédikt Tran, Chris Markiewicz, and Adam Turner in :gh:`118761`.) - +* :func:`~base64.b16decode` is now up to six times faster. + (Contributed by Bénédikt Tran, Chris Markiewicz, and Adam Turner + in :gh:`118761`.) -gc --- - -* The new :ref:`incremental garbage collector ` - means that maximum pause times are reduced - by an order of magnitude or more for larger heaps. - (Contributed by Mark Shannon in :gh:`108362`.) - -io +bdb --- -* :mod:`io` which provides the built-in :func:`open` makes less system calls - when opening regular files as well as reading whole files. Reading a small - operating system cached file in full is up to 15% faster. - :func:`pathlib.Path.read_bytes` has the most optimizations for reading a - file's bytes in full. (Contributed by Cody Maloney and Victor Stinner in - :gh:`120754` and :gh:`90102`.) - - -uuid ----- - -* Improve generation of :class:`~uuid.UUID` objects via their dedicated - functions: - - * :func:`~uuid.uuid3` and :func:`~uuid.uuid5` are both roughly 40% faster - for 16-byte names and 20% faster for 1024-byte names. Performance for - longer names remains unchanged. - * :func:`~uuid.uuid4` is 30% faster. - - (Contributed by Bénédikt Tran in :gh:`128150`.) +* The basic debugger now has a :mod:`sys.monitoring`-based backend, + which can be selected via the passing ``'monitoring'`` + to the :class:`~bdb.Bdb` class's new *backend* parameter. + (Contributed by Tian Gao in :gh:`124533`.) -zlib ----- -* On Windows, ``zlib-ng`` is now used as the implementation of the - :mod:`zlib` module. This should produce compatible and comparable - results with better performance, though it is worth noting that - ``zlib.Z_BEST_SPEED`` (1) may result in significantly less - compression than the previous implementation (while also significantly - reducing the time taken to compress). - (Contributed by Steve Dower in :gh:`91349`.) +difflib +------- +* The :func:`~difflib.IS_LINE_JUNK` function is now up to twice as fast. + (Contributed by Adam Turner and Semyon Moroz in :gh:`130167`.) -Deprecated -========== -* :mod:`argparse`: +gc +-- - * Passing the undocumented keyword argument *prefix_chars* to - :meth:`~argparse.ArgumentParser.add_argument_group` is now - deprecated. - (Contributed by Savannah Ostrowski in :gh:`125563`.) - * Deprecated the :class:`argparse.FileType` type converter. - Anything with resource management should be done downstream after the - arguments are parsed. - (Contributed by Serhiy Storchaka in :gh:`58032`.) +* **From Python 3.14.5 onwards:** -* :mod:`asyncio`: + Python 3.14.0-3.14.4 shipped with a new incremental garbage collector. + However, due to a number of `reports + `__ + of significant memory pressure in production environments, + it has been reverted back to the generational GC from 3.13. + This is the GC now used in Python 3.14.5 and later. - * :func:`!asyncio.iscoroutinefunction` is deprecated - and will be removed in Python 3.16; - use :func:`inspect.iscoroutinefunction` instead. - (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) +* **Previously in Python 3.14.0-3.14.4:** - * :mod:`asyncio` policy system is deprecated and will be removed in Python 3.16. - In particular, the following classes and functions are deprecated: + The new :ref:`incremental garbage collector ` + means that maximum pause times are reduced + by an order of magnitude or more for larger heaps. - * :class:`asyncio.AbstractEventLoopPolicy` - * :class:`asyncio.DefaultEventLoopPolicy` - * :class:`asyncio.WindowsSelectorEventLoopPolicy` - * :class:`asyncio.WindowsProactorEventLoopPolicy` - * :func:`asyncio.get_event_loop_policy` - * :func:`asyncio.set_event_loop_policy` + Because of this optimization, the meaning of the results of + :meth:`~gc.get_threshold` and :meth:`~gc.set_threshold` have changed, + along with :meth:`~gc.get_count` and :meth:`~gc.get_stats`. - Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with - *loop_factory* to use the desired event loop implementation. + - For backwards compatibility, :meth:`~gc.get_threshold` continues to return + a three-item tuple. + The first value is the threshold for young collections, as before; + the second value determines the rate at which the old collection is scanned + (the default is 10, and higher values mean that the old collection + is scanned more slowly). + The third value is now meaningless and is always zero. - For example, to use :class:`asyncio.SelectorEventLoop` on Windows:: + - :meth:`~gc.set_threshold` now ignores any items after the second. - import asyncio + - :meth:`~gc.get_count` and :meth:`~gc.get_stats` continue to return + the same format of results. + The only difference is that instead of the results referring to + the young, aging and old generations, + the results refer to the young generation + and the aging and collecting spaces of the old generation. - async def main(): - ... + In summary, code that attempted to manipulate the behavior of the cycle GC + may not work exactly as intended, but it is very unlikely to be harmful. + All other code will work just fine. - asyncio.run(main(), loop_factory=asyncio.SelectorEventLoop) + (Contributed by Mark Shannon in :gh:`108362`.) - (Contributed by Kumar Aditya in :gh:`127949`.) -* :mod:`builtins`: - Passing a complex number as the *real* or *imag* argument in the - :func:`complex` constructor is now deprecated; it should only be passed - as a single positional argument. - (Contributed by Serhiy Storchaka in :gh:`109218`.) +io +--- -* :mod:`codecs`: - :func:`codecs.open` is now deprecated. Use :func:`open` instead. - (Contributed by Inada Naoki in :gh:`133036`.) +* Opening and reading files now executes fewer system calls. + Reading a small operating system cached file in full is up to 15% faster. + (Contributed by Cody Maloney and Victor Stinner + in :gh:`120754` and :gh:`90102`.) -* :mod:`ctypes`: - * On non-Windows platforms, setting :attr:`.Structure._pack_` to use a - MSVC-compatible default memory layout is deprecated in favor of setting - :attr:`.Structure._layout_` to ``'ms'``. - (Contributed by Petr Viktorin in :gh:`131747`.) +pathlib +------- - * Calling :func:`ctypes.POINTER` on a string is deprecated. - Use :ref:`ctypes-incomplete-types` for self-referential structures. - Also, the internal ``ctypes._pointer_type_cache`` is deprecated. - See :func:`ctypes.POINTER` for updated implementation details. - (Contributed by Sergey Myrianov in :gh:`100926`.) +* :func:`Path.read_bytes ` now uses unbuffered mode + to open files, which is between 9% and 17% faster to read in full. + (Contributed by Cody Maloney in :gh:`120754`.) -* :mod:`functools`: - Calling the Python implementation of :func:`functools.reduce` with *function* - or *sequence* as keyword arguments is now deprecated. - (Contributed by Kirill Podoprigora in :gh:`121676`.) -* :mod:`logging`: - Support for custom logging handlers with the *strm* argument is deprecated - and scheduled for removal in Python 3.16. Define handlers with the *stream* - argument instead. (Contributed by Mariusz Felisiak in :gh:`115032`.) +pdb +--- -* :mod:`mimetypes`: - Valid extensions start with a '.' or are empty for - :meth:`mimetypes.MimeTypes.add_type`. - Undotted extensions are deprecated and will - raise a :exc:`ValueError` in Python 3.16. - (Contributed by Hugo van Kemenade in :gh:`75223`.) +* :mod:`pdb` now supports two backends, based on either + :func:`sys.settrace` or :mod:`sys.monitoring`. + Using the :ref:`pdb CLI ` or :func:`breakpoint` + will always use the :mod:`sys.monitoring` backend. + Explicitly instantiating :class:`pdb.Pdb` and its derived classes + will use the :func:`sys.settrace` backend by default, which is configurable. + (Contributed by Tian Gao in :gh:`124533`.) -* :mod:`!nturl2path`: This module is now deprecated. Call - :func:`urllib.request.url2pathname` and :func:`~urllib.request.pathname2url` - instead. - (Contributed by Barney Gale in :gh:`125866`.) -* :mod:`os`: - :term:`Soft deprecate ` :func:`os.popen` and - :func:`os.spawn* ` functions. They should no longer be used to - write new code. The :mod:`subprocess` module is recommended instead. - (Contributed by Victor Stinner in :gh:`120743`.) +textwrap +-------- -* :mod:`pathlib`: - :meth:`!pathlib.PurePath.as_uri` is deprecated and will be removed in Python - 3.19. Use :meth:`pathlib.Path.as_uri` instead. - (Contributed by Barney Gale in :gh:`123599`.) +* Optimize the :func:`~textwrap.dedent` function, improving performance by + an average of 2.4x, with larger improvements for bigger inputs, + and fix a bug with incomplete normalization of blank lines with whitespace + characters other than space and tab. -* :mod:`pdb`: - The undocumented ``pdb.Pdb.curframe_locals`` attribute is now a deprecated - read-only property. The low overhead dynamic frame locals access added in - Python 3.13 by PEP 667 means the frame locals cache reference previously - stored in this attribute is no longer needed. Derived debuggers should access - ``pdb.Pdb.curframe.f_locals`` directly in Python 3.13 and later versions. - (Contributed by Tian Gao in :gh:`124369` and :gh:`125951`.) -* :mod:`symtable`: - Deprecate :meth:`symtable.Class.get_methods` due to the lack of interest. - (Contributed by Bénédikt Tran in :gh:`119698`.) +uuid +---- -* :mod:`tkinter`: - The :class:`!tkinter.Variable` methods :meth:`!trace_variable`, - :meth:`!trace_vdelete` and :meth:`!trace_vinfo` are now deprecated. - Use :meth:`!trace_add`, :meth:`!trace_remove` and :meth:`!trace_info` - instead. - (Contributed by Serhiy Storchaka in :gh:`120220`.) +* :func:`~uuid.uuid3` and :func:`~uuid.uuid5` are now both roughly 40% faster + for 16-byte names and 20% faster for 1024-byte names. + Performance for longer names remains unchanged. + (Contributed by Bénédikt Tran in :gh:`128150`.) -* :mod:`urllib.parse`: - Accepting objects with false values (like ``0`` and ``[]``) except empty - strings, byte-like objects and ``None`` in :mod:`urllib.parse` functions - :func:`~urllib.parse.parse_qsl` and :func:`~urllib.parse.parse_qs` is now - deprecated. - (Contributed by Serhiy Storchaka in :gh:`116897`.) +* :func:`~uuid.uuid4` is now c. 30% faster. + (Contributed by Bénédikt Tran in :gh:`128150`.) -.. Add deprecations above alphabetically, not here at the end. -.. include:: ../deprecations/pending-removal-in-3.15.rst +zlib +---- -.. include:: ../deprecations/pending-removal-in-3.16.rst +* On Windows, `zlib-ng `__ + is now used as the implementation of the :mod:`zlib` module + in the default binaries. + There are no known incompatibilities between ``zlib-ng`` + and the previously-used ``zlib`` implementation. + This should result in better performance at all compression levels. -.. include:: ../deprecations/pending-removal-in-3.17.rst + It is worth noting that ``zlib.Z_BEST_SPEED`` (``1``) may result in + significantly less compression than the previous implementation, + whilst also significantly reducing the time taken to compress. -.. include:: ../deprecations/pending-removal-in-3.19.rst + (Contributed by Steve Dower in :gh:`91349`.) -.. include:: ../deprecations/pending-removal-in-future.rst Removed ======= @@ -2487,86 +2334,102 @@ argparse -------- * Remove the *type*, *choices*, and *metavar* parameters - of :class:`!argparse.BooleanOptionalAction`. - They were deprecated since 3.12. - -* Calling :meth:`~argparse.ArgumentParser.add_argument_group` on an argument - group, and calling :meth:`~argparse.ArgumentParser.add_argument_group` or - :meth:`~argparse.ArgumentParser.add_mutually_exclusive_group` on a mutually - exclusive group now raise exceptions. This nesting was never supported, - often failed to work correctly, and was unintentionally exposed through - inheritance. This functionality has been deprecated since Python 3.11. + of :class:`!BooleanOptionalAction`. + These have been deprecated since Python 3.12. + (Contributed by Nikita Sobolev in :gh:`118805`.) + +* Calling :meth:`~argparse.ArgumentParser.add_argument_group` + on an argument group now raises a :exc:`ValueError`. + Similarly, :meth:`~argparse.ArgumentParser.add_argument_group` + or :meth:`~argparse.ArgumentParser.add_mutually_exclusive_group` + on a mutually exclusive group now both raise :exc:`ValueError`\ s. + This 'nesting' was never supported, often failed to work correctly, + and was unintentionally exposed through inheritance. + This functionality has been deprecated since Python 3.11. (Contributed by Savannah Ostrowski in :gh:`127186`.) + ast --- -* Remove the following classes. They were all deprecated since Python 3.8, - and have emitted deprecation warnings since Python 3.12: +* Remove the following classes, which have been deprecated aliases of + :class:`~ast.Constant` since Python 3.8 and have emitted + deprecation warnings since Python 3.12: + + * :class:`!Bytes` + * :class:`!Ellipsis` + * :class:`!NameConstant` + * :class:`!Num` + * :class:`!Str` - * :class:`!ast.Bytes` - * :class:`!ast.Ellipsis` - * :class:`!ast.NameConstant` - * :class:`!ast.Num` - * :class:`!ast.Str` + As a consequence of these removals, user-defined ``visit_Num``, ``visit_Str``, + ``visit_Bytes``, ``visit_NameConstant`` and ``visit_Ellipsis`` methods + on custom :class:`~ast.NodeVisitor` subclasses will no longer be called + when the :class:`!NodeVisitor` subclass is visiting an AST. + Define a ``visit_Constant`` method instead. - Use :class:`ast.Constant` instead. As a consequence of these removals, - user-defined ``visit_Num``, ``visit_Str``, ``visit_Bytes``, - ``visit_NameConstant`` and ``visit_Ellipsis`` methods on custom - :class:`ast.NodeVisitor` subclasses will no longer be called when the - :class:`!NodeVisitor` subclass is visiting an AST. Define a ``visit_Constant`` - method instead. + (Contributed by Alex Waygood in :gh:`119562`.) - Also, remove the following deprecated properties on :class:`ast.Constant`, +* Remove the following deprecated properties on :class:`ast.Constant`, which were present for compatibility with the now-removed AST classes: - * :attr:`!ast.Constant.n` - * :attr:`!ast.Constant.s` + * :attr:`!Constant.n` + * :attr:`!Constant.s` - Use :attr:`!ast.Constant.value` instead. + Use :attr:`!Constant.value` instead. (Contributed by Alex Waygood in :gh:`119562`.) + asyncio ------- -* Remove the following classes and functions. They were all deprecated and - emitted deprecation warnings since Python 3.12: +* Remove the following classes, methods, and functions, + which have been deprecated since Python 3.12: - * :func:`!asyncio.get_child_watcher` - * :func:`!asyncio.set_child_watcher` - * :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` - * :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` - * :class:`!asyncio.AbstractChildWatcher` - * :class:`!asyncio.FastChildWatcher` - * :class:`!asyncio.MultiLoopChildWatcher` - * :class:`!asyncio.PidfdChildWatcher` - * :class:`!asyncio.SafeChildWatcher` - * :class:`!asyncio.ThreadedChildWatcher` + * :class:`!AbstractChildWatcher` + * :class:`!FastChildWatcher` + * :class:`!MultiLoopChildWatcher` + * :class:`!PidfdChildWatcher` + * :class:`!SafeChildWatcher` + * :class:`!ThreadedChildWatcher` + * :meth:`!AbstractEventLoopPolicy.get_child_watcher` + * :meth:`!AbstractEventLoopPolicy.set_child_watcher` + * :func:`!get_child_watcher` + * :func:`!set_child_watcher` (Contributed by Kumar Aditya in :gh:`120804`.) -* Removed implicit creation of event loop by :func:`asyncio.get_event_loop`. - It now raises a :exc:`RuntimeError` if there is no current event loop. +* :func:`asyncio.get_event_loop` now raises a :exc:`RuntimeError` + if there is no current event loop, + and no longer implicitly creates an event loop. + (Contributed by Kumar Aditya in :gh:`126353`.) + .. TODO: move these patterns to the asyncio docs? + quite long for What's New + There's a few patterns that use :func:`asyncio.get_event_loop`, most of them can be replaced with :func:`asyncio.run`. If you're running an async function, simply use :func:`asyncio.run`. - Before:: + Before: - async def main(): - ... + .. code:: python + async def main(): + ... - loop = asyncio.get_event_loop() - try: - loop.run_until_complete(main()) - finally: - loop.close() - After:: + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(main()) + finally: + loop.close() + + After: + + .. code:: python async def main(): ... @@ -2577,305 +2440,449 @@ asyncio and then run forever, use :func:`asyncio.run` and an :class:`asyncio.Event`. - Before:: - - def start_server(loop): - ... - - loop = asyncio.get_event_loop() - try: - start_server(loop) - loop.run_forever() - finally: - loop.close() - - After:: + Before: - def start_server(loop): - ... + .. code:: python - async def main(): - start_server(asyncio.get_running_loop()) - await asyncio.Event().wait() + def start_server(loop): ... - asyncio.run(main()) + loop = asyncio.get_event_loop() + try: + start_server(loop) + loop.run_forever() + finally: + loop.close() - If you need to run something in an event loop, then run some blocking - code around it, use :class:`asyncio.Runner`. + After: - Before:: + .. code:: python - async def operation_one(): - ... + def start_server(loop): ... - def blocking_code(): - ... + async def main(): + start_server(asyncio.get_running_loop()) + await asyncio.Event().wait() - async def operation_two(): - ... + asyncio.run(main()) - loop = asyncio.get_event_loop() - try: - loop.run_until_complete(operation_one()) - blocking_code() - loop.run_until_complete(operation_two()) - finally: - loop.close() + If you need to run something in an event loop, then run some blocking + code around it, use :class:`asyncio.Runner`. - After:: + Before: - async def operation_one(): - ... + .. code:: python - def blocking_code(): - ... + async def operation_one(): ... + def blocking_code(): ... + async def operation_two(): ... - async def operation_two(): - ... + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(operation_one()) + blocking_code() + loop.run_until_complete(operation_two()) + finally: + loop.close() - with asyncio.Runner() as runner: - runner.run(operation_one()) - blocking_code() - runner.run(operation_two()) + After: + .. code:: python + async def operation_one(): ... + def blocking_code(): ... + async def operation_two(): ... -collections.abc ---------------- + with asyncio.Runner() as runner: + runner.run(operation_one()) + blocking_code() + runner.run(operation_two()) -* Remove :class:`!collections.abc.ByteString`. It had previously raised a - :exc:`DeprecationWarning` since Python 3.12. email ----- -* Remove the *isdst* parameter from :func:`email.utils.localtime`. +* Remove :func:`email.utils.localtime`'s *isdst* parameter, + which was deprecated in and has been ignored since Python 3.12. (Contributed by Hugo van Kemenade in :gh:`118798`.) -importlib ---------- - -* Remove deprecated :mod:`importlib.abc` classes: - * :class:`!importlib.abc.ResourceReader` - * :class:`!importlib.abc.Traversable` - * :class:`!importlib.abc.TraversableResources` +importlib.abc +------------- - Use :mod:`importlib.resources.abc` classes instead: +* Remove deprecated :mod:`importlib.abc` classes: - * :class:`importlib.resources.abc.Traversable` - * :class:`importlib.resources.abc.TraversableResources` + * :class:`!ResourceReader` + (use :class:`~importlib.resources.abc.TraversableResources`) + * :class:`!Traversable` + (use :class:`~importlib.resources.abc.Traversable`) + * :class:`!TraversableResources` + (use :class:`~importlib.resources.abc.TraversableResources`) (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) + itertools --------- -* Remove :mod:`itertools` support for copy, deepcopy, and pickle operations. - These had previously raised a :exc:`DeprecationWarning` since Python 3.12. +* Remove support for copy, deepcopy, and pickle operations + from :mod:`itertools` iterators. + These have emitted a :exc:`DeprecationWarning` since Python 3.12. (Contributed by Raymond Hettinger in :gh:`101588`.) + pathlib ------- -* Remove support for passing additional keyword arguments to - :class:`pathlib.Path`. In previous versions, any such arguments are ignored. +* Remove support for passing additional keyword arguments + to :class:`~pathlib.Path`. + In previous versions, any such arguments are ignored. + (Contributed by Barney Gale in :gh:`74033`.) + * Remove support for passing additional positional arguments to - :meth:`pathlib.PurePath.relative_to` and - :meth:`~pathlib.PurePath.is_relative_to`. In previous versions, any such - arguments are joined onto *other*. + :meth:`.PurePath.relative_to` and :meth:`~pathlib.PurePath.is_relative_to`. + In previous versions, any such arguments are joined onto *other*. + (Contributed by Barney Gale in :gh:`78707`.) + pkgutil ------- -* Remove deprecated :func:`!pkgutil.get_loader` and :func:`!pkgutil.find_loader`. - These had previously raised a :exc:`DeprecationWarning` since Python 3.12. +* Remove the :func:`!get_loader` and :func:`!find_loader` functions, + which have been deprecated since Python 3.12. (Contributed by Bénédikt Tran in :gh:`97850`.) + pty --- -* Remove deprecated :func:`!pty.master_open` and :func:`!pty.slave_open`. - They had previously raised a :exc:`DeprecationWarning` since Python 3.12. +* Remove the :func:`!master_open` and :func:`!slave_open` functions, + which have been deprecated since Python 3.12. Use :func:`pty.openpty` instead. (Contributed by Nikita Sobolev in :gh:`118824`.) + sqlite3 ------- -* Remove :data:`!version` and :data:`!version_info` from :mod:`sqlite3`; +* Remove :data:`!version` and :data:`!version_info` from + the :mod:`sqlite3` module; use :data:`~sqlite3.sqlite_version` and :data:`~sqlite3.sqlite_version_info` for the actual version number of the runtime SQLite library. (Contributed by Hugo van Kemenade in :gh:`118924`.) -* Disallow using a sequence of parameters with named placeholders. - This had previously raised a :exc:`DeprecationWarning` since Python 3.12; - it will now raise a :exc:`sqlite3.ProgrammingError`. +* Using a sequence of parameters with named placeholders now + raises a :exc:`~sqlite3.ProgrammingError`, + having been deprecated since Python 3.12. (Contributed by Erlend E. Aasland in :gh:`118928` and :gh:`101693`.) -typing ------- - -* Remove :class:`!typing.ByteString`. It had previously raised a - :exc:`DeprecationWarning` since Python 3.12. - -* :class:`typing.TypeAliasType` now supports star unpacking. urllib ------ -* Remove deprecated :class:`!Quoter` class from :mod:`urllib.parse`. - It had previously raised a :exc:`DeprecationWarning` since Python 3.11. +* Remove the :class:`!Quoter` class from :mod:`urllib.parse`, + which has been deprecated since Python 3.11. (Contributed by Nikita Sobolev in :gh:`118827`.) -* Remove deprecated :class:`!URLopener` and :class:`!FancyURLopener` classes - from :mod:`urllib.request`. They had previously raised a - :exc:`DeprecationWarning` since Python 3.3. - - ``myopener.open()`` can be replaced with :func:`~urllib.request.urlopen`, - and ``myopener.retrieve()`` can be replaced with - :func:`~urllib.request.urlretrieve`. Customizations to the opener - classes can be replaced by passing customized handlers to - :func:`~urllib.request.build_opener`. - (Contributed by Barney Gale in :gh:`84850`.) -Others ------- +* Remove the :class:`!URLopener` and :class:`!FancyURLopener` classes + from :mod:`urllib.request`, + which have been deprecated since Python 3.3. -* Using :data:`NotImplemented` in a boolean context will now raise a :exc:`TypeError`. - It had previously raised a :exc:`DeprecationWarning` since Python 3.9. (Contributed - by Jelle Zijlstra in :gh:`118767`.) + ``myopener.open()`` can be replaced with :func:`~urllib.request.urlopen`. + ``myopener.retrieve()`` can be replaced with + :func:`~urllib.request.urlretrieve`. + Customisations to the opener classes can be replaced by passing + customized handlers to :func:`~urllib.request.build_opener`. + (Contributed by Barney Gale in :gh:`84850`.) -* The :func:`int` built-in no longer delegates to - :meth:`~object.__trunc__`. Classes that want to support conversion to - integer must implement either :meth:`~object.__int__` or - :meth:`~object.__index__`. (Contributed by Mark Dickinson in :gh:`119743`.) +Deprecated +========== -CPython bytecode changes -======================== +New deprecations +---------------- -* Replaced the opcode ``BINARY_SUBSCR`` by :opcode:`BINARY_OP` with oparg ``NB_SUBSCR``. - (Contributed by Irit Katriel in :gh:`100239`.) +* Passing a complex number as the *real* or *imag* argument in the + :func:`complex` constructor is now deprecated; + complex numbers should only be passed as a single positional argument. + (Contributed by Serhiy Storchaka in :gh:`109218`.) -Porting to Python 3.14 -====================== +* :mod:`argparse`: -This section lists previously described changes and other bugfixes -that may require changes to your code. + * Passing the undocumented keyword argument *prefix_chars* to the + :meth:`~argparse.ArgumentParser.add_argument_group` method is now deprecated. + (Contributed by Savannah Ostrowski in :gh:`125563`.) -Changes in the Python API -------------------------- + * Deprecated the :class:`argparse.FileType` type converter. + Anything relating to resource management should be handled + downstream, after the arguments have been parsed. + (Contributed by Serhiy Storchaka in :gh:`58032`.) -* :class:`functools.partial` is now a method descriptor. - Wrap it in :func:`staticmethod` if you want to preserve the old behavior. - (Contributed by Serhiy Storchaka and Dominykas Grigonis in :gh:`121027`.) +* :mod:`asyncio`: -* The :ref:`garbage collector is now incremental `, - which means that the behavior of :func:`gc.collect` changes slightly: + * The :func:`!asyncio.iscoroutinefunction` is now deprecated + and will be removed in Python 3.16; + use :func:`inspect.iscoroutinefunction` instead. + (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) - * ``gc.collect(1)``: Performs an increment of garbage collection, - rather than collecting generation 1. - * Other calls to :func:`!gc.collect` are unchanged. + * The :mod:`asyncio` policy system is deprecated + and will be removed in Python 3.16. + In particular, the following classes and functions are deprecated: -* The :func:`locale.nl_langinfo` function now sets temporarily the ``LC_CTYPE`` - locale in some cases. - This temporary change affects other threads. - (Contributed by Serhiy Storchaka in :gh:`69998`.) + * :class:`asyncio.AbstractEventLoopPolicy` + * :class:`asyncio.DefaultEventLoopPolicy` + * :class:`asyncio.WindowsSelectorEventLoopPolicy` + * :class:`asyncio.WindowsProactorEventLoopPolicy` + * :func:`asyncio.get_event_loop_policy` + * :func:`asyncio.set_event_loop_policy` -* :class:`types.UnionType` is now an alias for :class:`typing.Union`, - causing changes in some behaviors. - See :ref:`above ` for more details. - (Contributed by Jelle Zijlstra in :gh:`105499`.) + Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with + the *loop_factory* argument to use the desired event loop implementation. -* The runtime behavior of annotations has changed in various ways; see - :ref:`above ` for details. While most code that interacts - with annotations should continue to work, some undocumented details may behave - differently. + For example, to use :class:`asyncio.SelectorEventLoop` on Windows: + .. code-block:: python -Build changes -============= + import asyncio -* GNU Autoconf 2.72 is now required to generate :file:`configure`. - (Contributed by Erlend Aasland in :gh:`115765`.) + async def main(): + ... -* ``#pragma``-based linking with ``python3*.lib`` can now be switched off - with :c:expr:`Py_NO_LINK_LIB`. (Contributed by Jean-Christophe - Fillion-Robin in :gh:`82909`.) + asyncio.run(main(), loop_factory=asyncio.SelectorEventLoop) -.. _whatsnew314-pep761: + (Contributed by Kumar Aditya in :gh:`127949`.) -PEP 761: Discontinuation of PGP signatures ------------------------------------------- +* :mod:`codecs`: + The :func:`codecs.open` function is now deprecated, + and will be removed in a future version of Python. + Use :func:`open` instead. + (Contributed by Inada Naoki in :gh:`133036`.) + +* :mod:`ctypes`: -PGP signatures will not be available for CPython 3.14 and onwards. -Users verifying artifacts must use `Sigstore verification materials`_ for -verifying CPython artifacts. This change in release process is specified -in :pep:`761`. + * On non-Windows platforms, setting :attr:`.Structure._pack_` to use a + MSVC-compatible default memory layout is now deprecated in favor of setting + :attr:`.Structure._layout_` to ``'ms'``, and will be removed in Python 3.19. + (Contributed by Petr Viktorin in :gh:`131747`.) -.. _Sigstore verification materials: https://www.python.org/downloads/metadata/sigstore/ + * Calling :func:`ctypes.POINTER` on a string is now deprecated. + Use :ref:`incomplete types ` + for self-referential structures. + Also, the internal ``ctypes._pointer_type_cache`` is deprecated. + See :func:`ctypes.POINTER` for updated implementation details. + (Contributed by Sergey Myrianov in :gh:`100926`.) + +* :mod:`functools`: + Calling the Python implementation of :func:`functools.reduce` with *function* + or *sequence* as keyword arguments is now deprecated; + the parameters will be made positional-only in Python 3.16. + (Contributed by Kirill Podoprigora in :gh:`121676`.) + +* :mod:`logging`: + Support for custom logging handlers with the *strm* argument + is now deprecated and scheduled for removal in Python 3.16. + Define handlers with the *stream* argument instead. + (Contributed by Mariusz Felisiak in :gh:`115032`.) + +* :mod:`mimetypes`: + Valid extensions are either empty or must start with '.' for + :meth:`mimetypes.MimeTypes.add_type`. + Undotted extensions are deprecated and will + raise a :exc:`ValueError` in Python 3.16. + (Contributed by Hugo van Kemenade in :gh:`75223`.) + +* :mod:`!nturl2path`: + This module is now deprecated. Call :func:`urllib.request.url2pathname` + and :func:`~urllib.request.pathname2url` instead. + (Contributed by Barney Gale in :gh:`125866`.) + +* :mod:`os`: + The :func:`os.popen` and :func:`os.spawn* ` functions + are now :term:`soft deprecated`. + They should no longer be used to write new code. + The :mod:`subprocess` module is recommended instead. + (Contributed by Victor Stinner in :gh:`120743`.) + +* :mod:`pathlib`: + :meth:`!pathlib.PurePath.as_uri` is now deprecated + and scheduled for removal in Python 3.19. + Use :meth:`pathlib.Path.as_uri` instead. + (Contributed by Barney Gale in :gh:`123599`.) + +* :mod:`pdb`: + The undocumented ``pdb.Pdb.curframe_locals`` attribute is now a deprecated + read-only property, which will be removed in a future version of Python. + The low overhead dynamic frame locals access added in Python 3.13 by :pep:`667` + means the frame locals cache reference previously stored in this attribute + is no longer needed. Derived debuggers should access + ``pdb.Pdb.curframe.f_locals`` directly in Python 3.13 and later versions. + (Contributed by Tian Gao in :gh:`124369` and :gh:`125951`.) + +* :mod:`symtable`: + Deprecate :meth:`!symtable.Class.get_methods` due to the lack of interest, + scheduled for removal in Python 3.16. + (Contributed by Bénédikt Tran in :gh:`119698`.) + +* :mod:`tkinter`: + The :class:`!tkinter.Variable` methods :meth:`!trace_variable`, + :meth:`!trace_vdelete` and :meth:`!trace_vinfo` are now deprecated. + Use :meth:`!trace_add`, :meth:`!trace_remove` and :meth:`!trace_info` instead. + (Contributed by Serhiy Storchaka in :gh:`120220`.) + +* :mod:`urllib.parse`: + Accepting objects with false values (like ``0`` and ``[]``) except empty + strings, bytes-like objects and ``None`` in :func:`~urllib.parse.parse_qsl` + and :func:`~urllib.parse.parse_qs` is now deprecated. + (Contributed by Serhiy Storchaka in :gh:`116897`.) + +.. Add deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/pending-removal-in-3.15.rst + +.. include:: ../deprecations/pending-removal-in-3.16.rst + +.. include:: ../deprecations/pending-removal-in-3.17.rst + +.. include:: ../deprecations/pending-removal-in-3.18.rst + +.. include:: ../deprecations/pending-removal-in-3.19.rst + +.. include:: ../deprecations/pending-removal-in-3.20.rst + +.. include:: ../deprecations/pending-removal-in-3.21.rst + +.. include:: ../deprecations/pending-removal-in-future.rst + + +CPython bytecode changes +======================== + +* Replaced the opcode :opcode:`!BINARY_SUBSCR` by the :opcode:`BINARY_OP` + opcode with the ``NB_SUBSCR`` oparg. + (Contributed by Irit Katriel in :gh:`100239`.) + +* Add the :opcode:`BUILD_INTERPOLATION` and :opcode:`BUILD_TEMPLATE` + opcodes to construct new :class:`~string.templatelib.Interpolation` + and :class:`~string.templatelib.Template` instances, respectively. + (Contributed by Lysandros Nikolaou and others in :gh:`132661`; + see also :ref:`PEP 750: Template strings `). + +* Remove the :opcode:`!BUILD_CONST_KEY_MAP` opcode. + Use :opcode:`BUILD_MAP` instead. + (Contributed by Mark Shannon in :gh:`122160`.) + +* Replace the :opcode:`!LOAD_ASSERTION_ERROR` opcode with + :opcode:`LOAD_COMMON_CONSTANT` and add support for loading + :exc:`NotImplementedError`. + +* Add the :opcode:`LOAD_FAST_BORROW` and :opcode:`LOAD_FAST_BORROW_LOAD_FAST_BORROW` + opcodes to reduce reference counting overhead when the interpreter can prove + that the reference in the frame outlives the reference loaded onto the stack. + (Contributed by Matt Page in :gh:`130704`.) + +* Add the :opcode:`LOAD_SMALL_INT` opcode, which pushes a small integer + equal to the ``oparg`` to the stack. + The :opcode:`!RETURN_CONST` opcode is removed as it is no longer used. + (Contributed by Mark Shannon in :gh:`125837`.) + +* Add the new :opcode:`LOAD_SPECIAL` instruction. + Generate code for :keyword:`with` and :keyword:`async with` statements + using the new instruction. + Removed the :opcode:`!BEFORE_WITH` and :opcode:`!BEFORE_ASYNC_WITH` instructions. + (Contributed by Mark Shannon in :gh:`120507`.) + +* Add the :opcode:`POP_ITER` opcode to support 'virtual' iterators. + (Contributed by Mark Shannon in :gh:`132554`.) + + +Pseudo-instructions +------------------- + +* Add the :opcode:`!ANNOTATIONS_PLACEHOLDER` pseudo instruction + to support partially executed module-level annotations with + :ref:`deferred evaluation of annotations `. + (Contributed by Jelle Zijlstra in :gh:`130907`.) + +* Add the :opcode:`!BINARY_OP_EXTEND` pseudo instruction, + which executes a pair of functions (guard and specialization functions) + accessed from the inline cache. + (Contributed by Irit Katriel in :gh:`100239`.) + +* Add three specializations for :opcode:`CALL_KW`; + :opcode:`!CALL_KW_PY` for calls to Python functions, + :opcode:`!CALL_KW_BOUND_METHOD` for calls to bound methods, and + :opcode:`!CALL_KW_NON_PY` for all other calls. + (Contributed by Mark Shannon in :gh:`118093`.) + +* Add the :opcode:`JUMP_IF_TRUE` and :opcode:`JUMP_IF_FALSE` pseudo instructions, + conditional jumps which do not impact the stack. + Replaced by the sequence ``COPY 1``, ``TO_BOOL``, ``POP_JUMP_IF_TRUE/FALSE``. + (Contributed by Irit Katriel in :gh:`124285`.) + +* Add the :opcode:`!LOAD_CONST_MORTAL` pseudo instruction. + (Contributed by Mark Shannon in :gh:`128685`.) + +* Add the :opcode:`!LOAD_CONST_IMMORTAL` pseudo instruction, + which does the same as :opcode:`!LOAD_CONST`, but is more efficient + for immortal objects. + (Contributed by Mark Shannon in :gh:`125837`.) + +* Add the :opcode:`NOT_TAKEN` pseudo instruction, used by :mod:`sys.monitoring` + to record branch events (such as :monitoring-event:`BRANCH_LEFT`). + (Contributed by Mark Shannon in :gh:`122548`.) C API changes ============= -New features ------------- +.. _whatsnew314-capi-config: -* Add :c:func:`PyLong_GetSign` function to get the sign of :class:`int` objects. - (Contributed by Sergey B Kirpichev in :gh:`116560`.) +Python configuration C API +-------------------------- -* Add a new :c:type:`PyUnicodeWriter` API to create a Python :class:`str` - object: +Add a :ref:`PyInitConfig C API ` to configure the Python +initialization without relying on C structures and the ability to make +ABI-compatible changes in the future. - * :c:func:`PyUnicodeWriter_Create` - * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful` - * :c:func:`PyUnicodeWriter_Discard` - * :c:func:`PyUnicodeWriter_Finish` - * :c:func:`PyUnicodeWriter_Format` - * :c:func:`PyUnicodeWriter_WriteASCII` - * :c:func:`PyUnicodeWriter_WriteChar` - * :c:func:`PyUnicodeWriter_WriteRepr` - * :c:func:`PyUnicodeWriter_WriteStr` - * :c:func:`PyUnicodeWriter_WriteSubstring` - * :c:func:`PyUnicodeWriter_WriteUCS4` - * :c:func:`PyUnicodeWriter_WriteUTF8` - * :c:func:`PyUnicodeWriter_WriteWideChar` +Complete the :pep:`587` :ref:`PyConfig C API ` by adding +:c:func:`PyInitConfig_AddModule` which can be used to add a built-in extension +module; a feature previously referred to as the "inittab". - (Contributed by Victor Stinner in :gh:`119182`.) +Add :c:func:`PyConfig_Get` and :c:func:`PyConfig_Set` functions to get and set +the current runtime configuration. -* Add :c:func:`PyIter_NextItem` to replace :c:func:`PyIter_Next`, - which has an ambiguous return value. - (Contributed by Irit Katriel and Erlend Aasland in :gh:`105201`.) +:pep:`587` 'Python Initialization Configuration' unified all the ways +to configure Python's initialization. This PEP also unifies the configuration +of Python's preinitialization and initialization in a single API. +Moreover, this PEP only provides a single choice to embed Python, +instead of having two 'Python' and 'Isolated' choices (PEP 587), +to further simplify the API. -* Add :c:func:`PyLong_IsPositive`, :c:func:`PyLong_IsNegative` - and :c:func:`PyLong_IsZero` for checking if :c:type:`PyLongObject` - is positive, negative, or zero, respectively. - (Contributed by James Roy and Sergey B Kirpichev in :gh:`126061`.) +The lower level PEP 587 PyConfig API remains available for use cases +with an intentionally higher level of coupling to CPython implementation details +(such as emulating the full functionality of CPython's CLI, including its +configuration mechanisms). -* Add new functions to convert C ```` numbers from/to Python - :class:`int`: +(Contributed by Victor Stinner in :gh:`107954`.) - * :c:func:`PyLong_AsInt32` - * :c:func:`PyLong_AsInt64` - * :c:func:`PyLong_AsUInt32` - * :c:func:`PyLong_AsUInt64` - * :c:func:`PyLong_FromInt32` - * :c:func:`PyLong_FromInt64` - * :c:func:`PyLong_FromUInt32` - * :c:func:`PyLong_FromUInt64` +.. seealso:: :pep:`741` and :pep:`587` - (Contributed by Victor Stinner in :gh:`120389`.) + +New features in the C API +------------------------- + +* Add :c:func:`Py_PACK_VERSION` and :c:func:`Py_PACK_FULL_VERSION`, + two new macros for bit-packing Python version numbers. + This is useful for comparisons with :c:var:`Py_Version` + or :c:macro:`PY_VERSION_HEX`. + (Contributed by Petr Viktorin in :gh:`128629`.) * Add :c:func:`PyBytes_Join(sep, iterable) ` function, similar to ``sep.join(iterable)`` in Python. (Contributed by Victor Stinner in :gh:`121645`.) -* Add :c:func:`Py_HashBuffer` to compute and return the hash value of a buffer. - (Contributed by Antoine Pitrou and Victor Stinner in :gh:`122854`.) - -* Add functions to get and set the current runtime Python configuration - (:pep:`741`): +* Add functions to manipulate the configuration of the current + runtime Python interpreter + (:ref:`PEP 741: Python configuration C API `): * :c:func:`PyConfig_Get` * :c:func:`PyConfig_GetInt` @@ -2884,7 +2891,8 @@ New features (Contributed by Victor Stinner in :gh:`107954`.) -* Add functions to configure the Python initialization (:pep:`741`): +* Add functions to configure Python initialization + (:ref:`PEP 741: Python configuration C API `): * :c:func:`Py_InitializeFromInitConfig` * :c:func:`PyInitConfig_AddModule` @@ -2903,7 +2911,50 @@ New features (Contributed by Victor Stinner in :gh:`107954`.) -* Add a new import and export API for Python :class:`int` objects (:pep:`757`): +* Add :c:func:`Py_fopen` function to open a file. + This works similarly to the standard C :c:func:`!fopen` function, + instead accepting a Python object for the *path* parameter + and setting an exception on error. + The corresponding new :c:func:`Py_fclose` function should be used + to close a file. + (Contributed by Victor Stinner in :gh:`127350`.) + +* Add :c:func:`Py_HashBuffer` to compute and return the hash value of a buffer. + (Contributed by Antoine Pitrou and Victor Stinner in :gh:`122854`.) + +* Add :c:func:`PyImport_ImportModuleAttr` and + :c:func:`PyImport_ImportModuleAttrString` helper functions to import a module + and get an attribute of the module. + (Contributed by Victor Stinner in :gh:`128911`.) + +* Add :c:func:`PyIter_NextItem` to replace :c:func:`PyIter_Next`, + which has an ambiguous return value. + (Contributed by Irit Katriel and Erlend Aasland in :gh:`105201`.) + +* Add :c:func:`PyLong_GetSign` function to get the sign of :class:`int` objects. + (Contributed by Sergey B Kirpichev in :gh:`116560`.) + +* Add :c:func:`PyLong_IsPositive`, :c:func:`PyLong_IsNegative` + and :c:func:`PyLong_IsZero` for checking if :c:type:`PyLongObject` + is positive, negative, or zero, respectively. + (Contributed by James Roy and Sergey B Kirpichev in :gh:`126061`.) + +* Add new functions to convert C ```` numbers to/from + Python :class:`int` objects: + + * :c:func:`PyLong_AsInt32` + * :c:func:`PyLong_AsInt64` + * :c:func:`PyLong_AsUInt32` + * :c:func:`PyLong_AsUInt64` + * :c:func:`PyLong_FromInt32` + * :c:func:`PyLong_FromInt64` + * :c:func:`PyLong_FromUInt32` + * :c:func:`PyLong_FromUInt64` + + (Contributed by Victor Stinner in :gh:`120389`.) + +* Add a new import and export API for Python :class:`int` objects + (:pep:`757`): * :c:func:`PyLong_GetNativeLayout` * :c:func:`PyLong_Export` @@ -2914,83 +2965,465 @@ New features (Contributed by Sergey B Kirpichev and Victor Stinner in :gh:`102471`.) -* Add :c:func:`PyType_GetBaseByToken` and :c:data:`Py_tp_token` slot for easier - superclass identification, which attempts to resolve the `type checking issue - `__ mentioned in :pep:`630`. - (Contributed in :gh:`124153`.) - -* Add :c:func:`PyUnicode_Equal` function to the limited C API: - test if two strings are equal. - (Contributed by Victor Stinner in :gh:`124502`.) +* Add :c:func:`PyMonitoring_FireBranchLeftEvent` and + :c:func:`PyMonitoring_FireBranchRightEvent` for generating + :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` + events, respectively. + (Contributed by Mark Shannon in :gh:`122548`.) * Add :c:func:`PyType_Freeze` function to make a type immutable. (Contributed by Victor Stinner in :gh:`121654`.) -* Add :c:func:`PyUnstable_Object_EnableDeferredRefcount` for enabling - deferred reference counting, as outlined in :pep:`703`. +* Add :c:func:`PyType_GetBaseByToken` and :c:data:`Py_tp_token` slot + for easier superclass identification, which attempts to resolve the + type checking issue mentioned in :pep:`PEP 630 <630#type-checking>`. + (Contributed in :gh:`124153`.) -* Add :c:func:`PyMonitoring_FireBranchLeftEvent` and - :c:func:`PyMonitoring_FireBranchRightEvent` for generating - :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` - events, respectively. +* Add a new :c:func:`PyUnicode_Equal` function to test if two + strings are equal. + The function is also added to the Limited C API. + (Contributed by Victor Stinner in :gh:`124502`.) -* Add :c:func:`Py_fopen` function to open a file. Similar to the - :c:func:`!fopen` function, but the *path* parameter is a Python object and an - exception is set on error. Add also :c:func:`Py_fclose` function to close a - file. - (Contributed by Victor Stinner in :gh:`127350`.) +* Add a new :c:type:`PyUnicodeWriter` API to create a Python :class:`str` + object, with the following functions: + + * :c:func:`PyUnicodeWriter_Create` + * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful` + * :c:func:`PyUnicodeWriter_Discard` + * :c:func:`PyUnicodeWriter_Finish` + * :c:func:`PyUnicodeWriter_Format` + * :c:func:`PyUnicodeWriter_WriteASCII` + * :c:func:`PyUnicodeWriter_WriteChar` + * :c:func:`PyUnicodeWriter_WriteRepr` + * :c:func:`PyUnicodeWriter_WriteStr` + * :c:func:`PyUnicodeWriter_WriteSubstring` + * :c:func:`PyUnicodeWriter_WriteUCS4` + * :c:func:`PyUnicodeWriter_WriteUTF8` + * :c:func:`PyUnicodeWriter_WriteWideChar` + + (Contributed by Victor Stinner in :gh:`119182`.) * The ``k`` and ``K`` formats in :c:func:`PyArg_ParseTuple` and similar functions now use :meth:`~object.__index__` if available, like all other integer formats. (Contributed by Serhiy Storchaka in :gh:`112068`.) -* Add macros :c:func:`Py_PACK_VERSION` and :c:func:`Py_PACK_FULL_VERSION` for - bit-packing Python version numbers. - (Contributed by Petr Viktorin in :gh:`128629`.) - -* Add :c:func:`PyUnstable_IsImmortal` for determining whether an object is :term:`immortal`, - for debugging purposes. - -* Add :c:func:`PyImport_ImportModuleAttr` and - :c:func:`PyImport_ImportModuleAttrString` helper functions to import a module - and get an attribute of the module. - (Contributed by Victor Stinner in :gh:`128911`.) +* Add support for a new ``p`` format unit in :c:func:`Py_BuildValue` + that produces a Python :class:`bool` object from a C integer. + (Contributed by Pablo Galindo in :issue:`45325`.) -* Add support for a new ``p`` format unit in :c:func:`Py_BuildValue` that allows to - take a C integer and produce a Python :class:`bool` object. (Contributed by - Pablo Galindo in :issue:`45325`.) +* Add :c:func:`PyUnstable_IsImmortal` for determining if + an object is :term:`immortal`, for debugging purposes. + (Contributed by Peter Bierma in :gh:`128509`.) -* Add :c:func:`PyUnstable_Object_IsUniqueReferencedTemporary` to determine if an object - is a unique temporary object on the interpreter's operand stack. This can - be used in some cases as a replacement for checking if :c:func:`Py_REFCNT` - is ``1`` for Python objects passed as arguments to C API functions. +* Add :c:func:`PyUnstable_Object_EnableDeferredRefcount` for enabling + deferred reference counting, as outlined in :pep:`703`. -* Add :c:func:`PyUnstable_Object_IsUniquelyReferenced` as a replacement for - ``Py_REFCNT(op) == 1`` on :term:`free threaded ` builds. +* Add :c:func:`PyUnstable_Object_IsUniquelyReferenced` as + a replacement for ``Py_REFCNT(op) == 1`` on :term:`free threaded + ` builds. (Contributed by Peter Bierma in :gh:`133140`.) +* Add :c:func:`PyUnstable_Object_IsUniqueReferencedTemporary` to + determine if an object is a unique temporary object on the + interpreter's operand stack. + This can be used in some cases as a replacement for checking + if :c:func:`Py_REFCNT` is ``1`` for Python objects passed + as arguments to C API functions. + (Contributed by Sam Gross in :gh:`133164`.) + Limited C API changes --------------------- -* In the limited C API 3.14 and newer, :c:func:`Py_TYPE` and - :c:func:`Py_REFCNT` are now implemented as an opaque function call to hide - implementation details. +* In the limited C API version 3.14 and newer, :c:func:`Py_TYPE` and + :c:func:`Py_REFCNT` are now implemented as an opaque function call + to hide implementation details. (Contributed by Victor Stinner in :gh:`120600` and :gh:`124127`.) * Remove the :c:macro:`PySequence_Fast_GET_SIZE`, - :c:macro:`PySequence_Fast_GET_ITEM` and :c:macro:`PySequence_Fast_ITEMS` - macros from the limited C API, since these macros never worked in the limited - C API. Keep :c:func:`PySequence_Fast` in the limited C API. + :c:macro:`PySequence_Fast_GET_ITEM`, + and :c:macro:`PySequence_Fast_ITEMS` + macros from the limited C API, since they have always been broken + in the limited C API. (Contributed by Victor Stinner in :gh:`91417`.) +.. _whatsnew314-c-api-removed: + +Removed C APIs +-------------- + +* Creating :c:data:`immutable types ` with + mutable bases was deprecated in Python 3.12, + and now raises a :exc:`TypeError`. + (Contributed by Nikita Sobolev in :gh:`119775`.) + +* Remove ``PyDictObject.ma_version_tag`` member, which was deprecated + in Python 3.12. + Use the :c:func:`PyDict_AddWatcher` API instead. + (Contributed by Sam Gross in :gh:`124296`.) + +* Remove the private ``_Py_InitializeMain()`` function. + It was a :term:`provisional API` added to Python 3.8 by :pep:`587`. + (Contributed by Victor Stinner in :gh:`129033`.) + +* Remove the undocumented APIs :c:macro:`!Py_C_RECURSION_LIMIT` + and :c:member:`!PyThreadState.c_recursion_remaining`. + These were added in 3.13 and have been removed without deprecation. + Use :c:func:`Py_EnterRecursiveCall` to guard against runaway + recursion in C code. + (Removed by Petr Viktorin in :gh:`133079`, see also :gh:`130396`.) + + +.. _whatsnew314-c-api-deprecated: + +Deprecated C APIs +----------------- + +* The :c:macro:`!Py_HUGE_VAL` macro is now :term:`soft deprecated`. + Use :c:macro:`!INFINITY` instead. + (Contributed by Sergey B Kirpichev in :gh:`120026`.) + +* The :c:macro:`!Py_IS_NAN`, :c:macro:`!Py_IS_INFINITY`, + and :c:macro:`!Py_IS_FINITE` macros are now :term:`soft deprecated`. + Use :c:macro:`!isnan`, :c:macro:`!isinf` and :c:macro:`!isfinite` + instead, available from :file:`math.h` since C99. + (Contributed by Sergey B Kirpichev in :gh:`119613`.) + +* Non-tuple sequences are now deprecated as argument for the ``(items)`` + format unit in :c:func:`PyArg_ParseTuple` and other :ref:`argument + parsing ` functions if *items* contains format units + which store a :ref:`borrowed buffer ` or a + :term:`borrowed reference`. + (Contributed by Serhiy Storchaka in :gh:`50333`.) + +* The ``_PyMonitoring_FireBranchEvent`` function is now deprecated + and should be replaced with calls to + :c:func:`PyMonitoring_FireBranchLeftEvent` and + :c:func:`PyMonitoring_FireBranchRightEvent`. + +* The previously undocumented function :c:func:`PySequence_In` is + now :term:`soft deprecated`. + Use :c:func:`PySequence_Contains` instead. + (Contributed by Yuki Kobayashi in :gh:`127896`.) + +.. Add C API deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/c-api-pending-removal-in-3.15.rst + +.. include:: ../deprecations/c-api-pending-removal-in-3.16.rst + +.. include:: ../deprecations/c-api-pending-removal-in-3.18.rst + +.. include:: ../deprecations/c-api-pending-removal-in-future.rst + + +.. _whatsnew314-build-changes: + +Build changes +============= + +* :pep:`776`: Emscripten is now an officially supported platform at + :pep:`tier 3 <11#tier-3>`. As a part of this effort, more than 25 bugs in + `Emscripten libc`__ were fixed. Emscripten now includes support + for :mod:`ctypes`, :mod:`termios`, and :mod:`fcntl`, as well as + experimental support for the new :ref:`default interactive shell + `. + (Contributed by R. Hood Chatham in :gh:`127146`, :gh:`127683`, and :gh:`136931`.) + + __ https://emscripten.org/docs/porting/emscripten-runtime-environment.html + +* Official Android binary releases are now provided on python.org__. + + __ https://www.python.org/downloads/android/ + +* GNU Autoconf 2.72 is now required to generate :file:`configure`. + (Contributed by Erlend Aasland in :gh:`115765`.) + +* ``wasm32-unknown-emscripten`` is now a :pep:`11` tier 3 platform. + (Contributed by R. Hood Chatham in :gh:`127146`, :gh:`127683`, and :gh:`136931`.) + +* ``#pragma``-based linking with ``python3*.lib`` can now be switched off + with :c:expr:`Py_NO_LINK_LIB`. + (Contributed by Jean-Christophe Fillion-Robin in :gh:`82909`.) + +* CPython now enables a set of recommended compiler options by default + for improved security. + Use the :option:`--disable-safety` :file:`configure` option to disable them, + or the :option:`--enable-slower-safety` option for a larger set + of compiler options, albeit with a performance cost. + +* The ``WITH_FREELISTS`` macro and ``--without-freelists`` :file:`configure` + option have been removed. + +* The new :file:`configure` option :option:`--with-tail-call-interp` + may be used to enable the experimental tail call interpreter. + See :ref:`whatsnew314-tail-call-interpreter` for further details. + +* To disable the new remote debugging support, use the + :option:`--without-remote-debug` :file:`configure` option. + This may be useful for security reasons. + +* iOS and macOS apps can now be configured to redirect ``stdout`` and + ``stderr`` content to the system log. + (Contributed by Russell Keith-Magee in :gh:`127592`.) + +* The iOS testbed is now able to stream test output while the test is running. + The testbed can also be used to run the test suite of projects other than + CPython itself. + (Contributed by Russell Keith-Magee in :gh:`127592`.) + + +.. _whatsnew314-build_details: + +:file:`build-details.json` +-------------------------- + +Installations of Python now contain a new file, :file:`build-details.json`. +This is a static JSON document containing build details for CPython, +to allow for introspection without needing to run code. +This is helpful for use-cases such as Python launchers, cross-compilation, +and so on. + +:file:`build-details.json` must be installed in the platform-independent +standard library directory. This corresponds to the :ref:`'stdlib' +` :mod:`sysconfig` installation path, +which can be found by running ``sysconfig.get_path('stdlib')``. + +.. seealso:: + :pep:`739` -- ``build-details.json`` 1.0 -- a static description file + for Python build details + + +.. _whatsnew314-no-more-pgp: + +Discontinuation of PGP signatures +--------------------------------- + +PGP (Pretty Good Privacy) signatures will not be provided +for releases of Python 3.14 or future versions. +To verify CPython artifacts, users must use `Sigstore verification materials +`__. +Releases have been signed using Sigstore_ since Python 3.11. + +This change in release process was specified in :pep:`761`. + +.. _Sigstore: https://www.sigstore.dev/ + + +.. _whatsnew314-free-threaded-now-supported: + +Free-threaded Python is officially supported +-------------------------------------------- + +The free-threaded build of Python is now supported and no longer experimental. +This is the start of `phase II `__ where +free-threaded Python is officially supported but still optional. + +The free-threading team are confident that the project is on the right path, +and appreciate the continued dedication from everyone working to make +free-threading ready for broader adoption across the Python community. + +With these recommendations and the acceptance of this PEP, the Python developer +community should broadly advertise that free-threading is a supported +Python build option now and into the future, and that it will not be removed +without a proper deprecation schedule. + +Any decision to transition to `phase III `__, +with free-threading as the default or sole build of Python is still undecided, +and dependent on many factors both within CPython itself and the community. +This decision is for the future. + +.. seealso:: + + :pep:`779` + + `PEP 779's acceptance `__ + + +.. _whatsnew314-jit-compiler: + +Binary releases for the experimental just-in-time compiler +---------------------------------------------------------- + +The official macOS and Windows release binaries now include an *experimental* +just-in-time (JIT) compiler. Although it is **not** recommended for production +use, it can be tested by setting :envvar:`PYTHON_JIT=1 ` as an +environment variable. Downstream source builds and redistributors can use the +:option:`--enable-experimental-jit=yes-off` configuration option for similar +behavior. + +The JIT is at an early stage and still in active development. As such, the +typical performance impact of enabling it can range from 10% slower to 20% +faster, depending on workload. To aid in testing and evaluation, a set of +introspection functions has been provided in the :data:`sys._jit` namespace. +:func:`sys._jit.is_available` can be used to determine if the current executable +supports JIT compilation, while :func:`sys._jit.is_enabled` can be used to tell +if JIT compilation has been enabled for the current process. + +Currently, the most significant missing functionality is that native debuggers +and profilers like ``gdb`` and ``perf`` are unable to unwind through JIT frames +(Python debuggers and profilers, like :mod:`pdb` or :mod:`profile`, continue to +work without modification). Free-threaded builds do not support JIT compilation. + +Please report any bugs or major performance regressions that you encounter! + +.. seealso:: :pep:`744` + + Porting to Python 3.14 ----------------------- +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + + +Changes in the Python API +------------------------- + +* On Unix platforms other than macOS, *forkserver* is now the default + :ref:`start method ` for :mod:`multiprocessing` + and :class:`~concurrent.futures.ProcessPoolExecutor`, instead of *fork*. + + See :ref:`(1) ` and + :ref:`(2) ` for details. + + If you encounter :exc:`NameError`\s or pickling errors coming out of + :mod:`multiprocessing` or :mod:`concurrent.futures`, see the + :ref:`forkserver restrictions `. + + This change does not affect Windows or macOS, where :ref:`'spawn' + ` remains the default start method. + +* :class:`functools.partial` is now a method descriptor. + Wrap it in :func:`staticmethod` if you want to preserve the old behavior. + (Contributed by Serhiy Storchaka and Dominykas Grigonis in :gh:`121027`.) + +* The :ref:`garbage collector is now incremental `, + which means that the behavior of :func:`gc.collect` changes slightly: + + * ``gc.collect(1)``: Performs an increment of garbage collection, + rather than collecting generation 1. + * Other calls to :func:`!gc.collect` are unchanged. + +* The :func:`locale.nl_langinfo` function now temporarily sets the ``LC_CTYPE`` + locale in some cases. + This temporary change affects other threads. + (Contributed by Serhiy Storchaka in :gh:`69998`.) + +* :class:`types.UnionType` is now an alias for :class:`typing.Union`, + causing changes in some behaviors. + See :ref:`above ` for more details. + (Contributed by Jelle Zijlstra in :gh:`105499`.) + +* The runtime behavior of annotations has changed in various ways; see + :ref:`above ` for details. While most code that interacts + with annotations should continue to work, some undocumented details may behave + differently. + +* As part of making the :mod:`mimetypes` CLI public, + it now exits with ``1`` on failure instead of ``0`` + and ``2`` on incorrect command-line parameters instead of ``1``. + Error messages are now printed to stderr. + +* The ``\B`` pattern in regular expression now matches the empty string + when given as the entire pattern, which may cause behavioural changes. + +* On FreeBSD, :data:`sys.platform` no longer contains the major version number. + + +.. _whatsnew314-porting-annotations: + +Changes in annotations (:pep:`649` and :pep:`749`) +-------------------------------------------------- + +This section contains guidance on changes that may be needed to annotations +or Python code that interacts with or introspects annotations, +due to the changes related to :ref:`deferred evaluation of annotations +`. + +In the majority of cases, working code from older versions of Python +will not require any changes. + + +Implications for annotated code +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you define annotations in your code (for example, for use with a static type +checker), then this change probably does not affect you: you can keep +writing annotations the same way you did with previous versions of Python. + +You will likely be able to remove quoted strings in annotations, which are frequently +used for forward references. Similarly, if you use ``from __future__ import annotations`` +to avoid having to write strings in annotations, you may well be able to +remove that import once you support only Python 3.14 and newer. +However, if you rely on third-party libraries that read annotations, +those libraries may need changes to support unquoted annotations before they +work as expected. + + +Implications for readers of ``__annotations__`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If your code reads the :attr:`~object.__annotations__` attribute on objects, +you may want to make changes in order to support code that relies on +deferred evaluation of annotations. +For example, you may want to use :func:`annotationlib.get_annotations` with +the :attr:`~annotationlib.Format.FORWARDREF` format, +as the :mod:`dataclasses` module now does. + +The external :pypi:`typing_extensions` package provides partial backports +of some of the functionality of the :mod:`annotationlib` module, +such as the :class:`~annotationlib.Format` enum and +the :func:`~annotationlib.get_annotations` function. +These can be used to write cross-version code that takes advantage of +the new behavior in Python 3.14. + + +Related changes +^^^^^^^^^^^^^^^ + +The changes in Python 3.14 are designed to rework how :attr:`!__annotations__` +works at runtime while minimizing breakage to code that contains +annotations in source code and to code that reads :attr:`!__annotations__`. +However, if you rely on undocumented details of the annotation behavior +or on private functions in the standard library, there are many ways in which +your code may not work in Python 3.14. +To safeguard your code against future changes, only use the documented +functionality of the :mod:`annotationlib` module. + +In particular, do not read annotations directly from the namespace dictionary +attribute of type objects. +Use :func:`annotationlib.get_annotate_from_class_namespace` during class +construction and :func:`annotationlib.get_annotations` afterwards. + +In previous releases, it was sometimes possible to access class annotations +from an instance of an annotated class. This behavior was undocumented +and accidental, and will no longer work in Python 3.14. + + +``from __future__ import annotations`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In Python 3.7, :pep:`563` introduced the ``from __future__ import annotations`` +:ref:`future statement `, which turns all annotations into strings. + +However, this statement is now deprecated and it is expected to be removed +in a future version of Python. +This removal will not happen until after Python 3.13 reaches its end of life +in 2029, being the last version of Python without support for deferred +evaluation of annotations. + +In Python 3.14, the behavior of code using ``from __future__ import annotations`` +is unchanged. + + +Changes in the C API +-------------------- * :c:func:`Py_Finalize` now deletes all interned strings. This - is backwards incompatible to any C-Extension that holds onto an interned + is backwards incompatible to any C extension that holds onto an interned string after a call to :c:func:`Py_Finalize` and is then reused after a call to :c:func:`Py_Initialize`. Any issues arising from this behavior will normally result in crashes during the execution of the subsequent call to @@ -3043,108 +3476,15 @@ Porting to Python 3.14 .. _pythoncapi-compat project: https://github.com/python/pythoncapi-compat/ -.. _whatsnew314-c-api-deprecated: - -Deprecated ----------- - -* The :c:macro:`!Py_HUGE_VAL` macro is :term:`soft deprecated`, - use :c:macro:`!Py_INFINITY` instead. - (Contributed by Sergey B Kirpichev in :gh:`120026`.) - -* Macros :c:macro:`!Py_IS_NAN`, :c:macro:`!Py_IS_INFINITY` - and :c:macro:`!Py_IS_FINITE` are :term:`soft deprecated`, - use instead :c:macro:`!isnan`, :c:macro:`!isinf` and - :c:macro:`!isfinite` available from :file:`math.h` - since C99. (Contributed by Sergey B Kirpichev in :gh:`119613`.) - -* Non-tuple sequences are deprecated as argument for the ``(items)`` - format unit in :c:func:`PyArg_ParseTuple` and other - :ref:`argument parsing ` functions if *items* contains - format units which store a :ref:`borrowed buffer ` - or a :term:`borrowed reference`. - (Contributed by Serhiy Storchaka in :gh:`50333`.) - -* The previously undocumented function :c:func:`PySequence_In` is :term:`soft deprecated`. - Use :c:func:`PySequence_Contains` instead. - (Contributed by Yuki Kobayashi in :gh:`127896`.) - -.. Add C API deprecations above alphabetically, not here at the end. - -* The ``PyMonitoring_FireBranchEvent`` function is deprecated and should - be replaced with calls to :c:func:`PyMonitoring_FireBranchLeftEvent` - and :c:func:`PyMonitoring_FireBranchRightEvent`. - -* The following private functions are deprecated and planned for removal in - Python 3.18: - - * :c:func:`!_PyBytes_Join`: use :c:func:`PyBytes_Join`. - * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. - * :c:func:`!_PyDict_Pop()`: use :c:func:`PyDict_Pop`. - * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. - * :c:func:`!_PyLong_FromDigits` and :c:func:`!_PyLong_New`: - use :c:func:`PyLongWriter_Create`. - * :c:func:`!_PyThreadState_UncheckedGet`: use :c:func:`PyThreadState_GetUnchecked`. - * :c:func:`!_PyUnicode_AsString`: use :c:func:`PyUnicode_AsUTF8`. - * :c:func:`!_PyUnicodeWriter_Init`: - replace ``_PyUnicodeWriter_Init(&writer)`` with - :c:func:`writer = PyUnicodeWriter_Create(0) `. - * :c:func:`!_PyUnicodeWriter_Finish`: - replace ``_PyUnicodeWriter_Finish(&writer)`` with - :c:func:`PyUnicodeWriter_Finish(writer) `. - * :c:func:`!_PyUnicodeWriter_Dealloc`: - replace ``_PyUnicodeWriter_Dealloc(&writer)`` with - :c:func:`PyUnicodeWriter_Discard(writer) `. - * :c:func:`!_PyUnicodeWriter_WriteChar`: - replace ``_PyUnicodeWriter_WriteChar(&writer, ch)`` with - :c:func:`PyUnicodeWriter_WriteChar(writer, ch) `. - * :c:func:`!_PyUnicodeWriter_WriteStr`: - replace ``_PyUnicodeWriter_WriteStr(&writer, str)`` with - :c:func:`PyUnicodeWriter_WriteStr(writer, str) `. - * :c:func:`!_PyUnicodeWriter_WriteSubstring`: - replace ``_PyUnicodeWriter_WriteSubstring(&writer, str, start, end)`` with - :c:func:`PyUnicodeWriter_WriteSubstring(writer, str, start, end) `. - * :c:func:`!_PyUnicodeWriter_WriteASCIIString`: - replace ``_PyUnicodeWriter_WriteASCIIString(&writer, str)`` with - :c:func:`PyUnicodeWriter_WriteASCII(writer, str) `. - * :c:func:`!_PyUnicodeWriter_WriteLatin1String`: - replace ``_PyUnicodeWriter_WriteLatin1String(&writer, str)`` with - :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) `. - * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. - * :c:func:`!_Py_fopen_obj`: use :c:func:`Py_fopen`. - - The `pythoncapi-compat project`_ can be used to get these new public - functions on Python 3.13 and older. - (Contributed by Victor Stinner in :gh:`128863`.) - -.. include:: ../deprecations/c-api-pending-removal-in-3.15.rst - -.. include:: ../deprecations/c-api-pending-removal-in-3.16.rst - -.. include:: ../deprecations/c-api-pending-removal-in-3.18.rst - -.. include:: ../deprecations/c-api-pending-removal-in-future.rst - - -.. _whatsnew314-c-api-removed: - -Removed -------- - -* Creating :c:data:`immutable types ` with mutable - bases was deprecated since 3.12 and now raises a :exc:`TypeError`. - -* Remove ``PyDictObject.ma_version_tag`` member which was deprecated since - Python 3.12. Use the :c:func:`PyDict_AddWatcher` API instead. - (Contributed by Sam Gross in :gh:`124296`.) +Notable changes in 3.14.5 +========================= -* Remove the private ``_Py_InitializeMain()`` function. It was a - :term:`provisional API` added to Python 3.8 by :pep:`587`. - (Contributed by Victor Stinner in :gh:`129033`.) +gc +-- -* The undocumented APIs :c:macro:`!Py_C_RECURSION_LIMIT` and - :c:member:`!PyThreadState.c_recursion_remaining`, added in 3.13, are removed - without a deprecation period. - Please use :c:func:`Py_EnterRecursiveCall` to guard against runaway recursion - in C code. - (Removed in :gh:`133079`, see also :gh:`130396`.) +* The incremental garbage collector shipped in Python 3.14.0-3.14.4 has been + reverted back to the generational garbage collector from 3.13, + due to a number of `reports + `__ + of significant memory pressure in production environments. + See :ref:`whatsnew314-incremental-gc` for details. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index e8e2c1ed6047bfe..db6903da9f63444 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -3,7 +3,7 @@ What's new in Python 3.15 **************************** -:Editor: TBD +:Editor: Hugo van Kemenade .. Rules for maintenance: @@ -56,8 +56,8 @@ For full details, see the :ref:`changelog `. so it's worth checking back even after reading earlier versions. -Summary --- release highlights -============================== +Summary -- Release highlights +============================= .. This section singles out the most important changes in Python 3.15. Brevity is key. @@ -65,116 +65,620 @@ Summary --- release highlights .. PEP-sized items next. +* :pep:`810`: :ref:`Explicit lazy imports for faster startup times + ` +* :pep:`814`: :ref:`Add frozendict built-in type + ` +* :pep:`661`: :ref:`Add sentinel built-in type + ` +* :pep:`799`: :ref:`A dedicated profiling package for organizing Python + profiling tools ` +* :pep:`799`: :ref:`Tachyon: High frequency statistical sampling profiler + ` +* :pep:`831`: :ref:`Frame pointers are enabled by default for improved + system-level observability ` +* :pep:`798`: :ref:`Unpacking in comprehensions + ` +* :pep:`686`: :ref:`Python now uses UTF-8 as the default encoding + ` +* :pep:`829`: :ref:`Package startup configuration files ` +* :pep:`728`: :ref:`TypedDict with typed extra items ` +* :pep:`747`: :ref:`Annotating type forms with TypeForm + ` +* :pep:`800`: Disjoint bases in the type system +* :pep:`782`: :ref:`A new PyBytesWriter C API to create a Python bytes object + ` +* :pep:`803`, :pep:`820 <820>`, :pep:`793 <793>`: + :ref:`Stable ABI for free-threaded builds ` and + related C API +* :pep:`788`: :ref:`Protection against finalization in the C API ` +* :ref:`The JIT compiler has been significantly upgraded ` +* :ref:`The official Windows 64-bit binaries now use the tail-calling interpreter + ` +* :ref:`Improved error messages ` +* :ref:`More color ` New features ============ +.. _whatsnew315-lazy-imports: + +:pep:`810`: Explicit lazy imports +--------------------------------- + +Large Python applications often suffer from slow startup times. A +significant contributor to this problem is the import system: when a module +is imported, Python must locate the file, read it from disk, compile it to +bytecode, and execute all top-level code. For applications with deep +dependency trees, this process can take seconds, even when most of the +imported code is never actually used during a particular run. + +Developers have worked around this by moving imports inside functions, using +:mod:`importlib` to load modules on demand, or restructuring code to avoid +unnecessary dependencies. These approaches work but make code harder to read +and maintain, scatter import statements throughout the codebase, and require +discipline to apply consistently. + +Python now provides a cleaner solution through explicit :keyword:`lazy` +imports using the new ``lazy`` soft keyword. When you mark an import as +lazy, Python defers the actual module loading until the imported name is +first used. This gives you the organizational benefits of declaring all +imports at the top of the file while only paying the loading cost for +modules you actually use. + +The ``lazy`` keyword works with both ``import`` and ``from ... import`` +statements. When you write ``lazy import heavy_module``, Python does not +immediately load the module. Instead, it creates a lightweight proxy object. +The actual module loading happens transparently when you first access the +name: + +.. code-block:: python + + lazy import json + lazy from pathlib import Path + + print("Starting up...") # json and pathlib not loaded yet + + data = json.loads('{"key": "value"}') # json loads here + p = Path(".") # pathlib loads here + +This mechanism is particularly useful for applications that import many +modules at the top level but may only use a subset of them in any given run. +The deferred loading reduces startup latency without requiring code +restructuring or conditional imports scattered throughout the codebase. + +In the case where loading a lazily imported module fails (for example, if +the module does not exist), Python raises the exception at the point of +first use rather than at import time. The associated traceback includes both +the location where the name was accessed and the original import statement, +making it straightforward to diagnose and debug the failure. + +For cases where you want to enable lazy loading globally without modifying +source code, Python provides the :option:`-X lazy_imports <-X>` command-line +option and the :envvar:`PYTHON_LAZY_IMPORTS` environment variable. Both +accept two values: ``all`` makes all imports lazy by default, and ``normal`` +(the default) respects the ``lazy`` keyword in source code. The +:func:`sys.set_lazy_imports` and :func:`sys.get_lazy_imports` functions allow +changing and querying this mode at runtime. + +For more selective control, :func:`sys.set_lazy_imports_filter` accepts a +callable that determines whether a specific module should be loaded lazily. +The filter receives three arguments: the importing module's name (or +``None``), the imported module's name, and the fromlist (or ``None`` for +regular imports). It should return ``True`` to allow the import to be lazy, +or ``False`` to force eager loading. This allows patterns like making only +your own application's modules lazy while keeping third-party dependencies +eager: + +.. code-block:: python + + import sys + + def myapp_filter(importing, imported, fromlist): + return imported.startswith("myapp.") + sys.set_lazy_imports_filter(myapp_filter) + sys.set_lazy_imports("all") + + import myapp.slow_module # lazy (matches filter) + import json # eager (does not match filter) + +The proxy type itself is available as :class:`types.LazyImportType` for code +that needs to detect lazy imports programmatically. + +There are some restrictions on where the ``lazy`` keyword can be used. Lazy +imports are only permitted at module scope; using ``lazy`` inside a +function, class body, or ``try``/``except``/``finally`` block raises a +:exc:`SyntaxError`. Neither star imports nor future imports can be lazy +(``lazy from module import *`` and ``lazy from __future__ import ...`` both +raise :exc:`SyntaxError`). + +For code that cannot use the ``lazy`` keyword directly (for example, when +supporting Python versions older than 3.15 while still using lazy +imports on 3.15+), a module can define +:attr:`~module.__lazy_modules__` as a container of fully qualified module +name strings. Regular ``import`` statements for those modules are then treated +as lazy, with the same semantics as the ``lazy`` keyword:: + + __lazy_modules__ = ["json", "pathlib"] + + import json # lazy + import os # still eager + +.. seealso:: :pep:`810` for the full specification and rationale. + +(Contributed by Pablo Galindo Salgado and Dino Viehland in :gh:`142349`.) + + +.. _whatsnew315-frozendict: + +:pep:`814`: Add frozendict built-in type +---------------------------------------- + +A new :term:`immutable` type, :class:`frozendict`, is added to the :mod:`builtins` module. +It does not allow modification after creation. A :class:`!frozendict` is not a subclass of ``dict``; +it inherits directly from ``object``. A :class:`!frozendict` is :term:`hashable` +as long as all of its keys and values are hashable. A :class:`!frozendict` preserves +insertion order, but comparison does not take order into account. + +For example:: + + >>> a = frozendict(x=1, y=2) + >>> a + frozendict({'x': 1, 'y': 2}) + >>> a['z'] = 3 + Traceback (most recent call last): + File "", line 1, in + a['z'] = 3 + ~^^^^^ + TypeError: 'frozendict' object does not support item assignment + >>> b = frozendict(y=2, x=1) + >>> hash(a) == hash(b) + True + >>> a == b + True + +The following standard library modules have been updated to accept +:class:`!frozendict`: :mod:`copy`, :mod:`decimal`, :mod:`json`, :mod:`marshal`, +:mod:`plistlib` (only for serialization), :mod:`pickle`, :mod:`pprint` and +:mod:`xml.etree.ElementTree`. + +:func:`eval` and :func:`exec` accept :class:`!frozendict` for *globals*, and +:func:`type` and :meth:`str.maketrans` accept :class:`!frozendict` for *dict*. + +Code checking for :class:`dict` type using ``isinstance(arg, dict)`` can be +updated to ``isinstance(arg, (dict, frozendict))`` to accept also the +:class:`!frozendict` type, or to ``isinstance(arg, collections.abc.Mapping)`` +to accept also other mapping types such as :class:`~types.MappingProxyType`. + +.. seealso:: :pep:`814` for the full specification and rationale. + +(Contributed by Victor Stinner and Donghee Na in :gh:`141510`.) + + +.. _whatsnew315-sentinel: + +:pep:`661`: Add sentinel built-in type +-------------------------------------- + +A new :class:`sentinel` type is added to the :mod:`builtins` module for +creating unique sentinel values with a concise representation. Sentinel +objects preserve identity when copied, support use in type expressions with +the ``|`` operator, and can be pickled when they are importable by module and +name. + +(PEP by Tal Einat; contributed by Jelle Zijlstra in :gh:`148829`.) + +.. seealso:: :pep:`661` for further details. + + +.. _whatsnew315-profiling-package: + +:pep:`799`: A dedicated profiling package +----------------------------------------- + +A new :mod:`profiling` module has been added to organize Python's built-in +profiling tools under a single, coherent namespace. This module contains: + +* :mod:`profiling.tracing`: deterministic function-call tracing (relocated from + ``cProfile``). +* :mod:`profiling.sampling`: a new statistical sampling profiler (named Tachyon). + +The ``cProfile`` module remains as an alias for backwards compatibility. +The :mod:`profile` module is deprecated and will be removed in Python 3.17. + +.. seealso:: :pep:`799` for further details. + +(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`138122`.) + + .. _whatsnew315-sampling-profiler: -High frequency statistical sampling profiler --------------------------------------------- +Tachyon: High frequency statistical sampling profiler +----------------------------------------------------- + +.. image:: ../../Lib/profiling/sampling/_assets/tachyon-logo.png + :alt: Tachyon profiler logo + :align: center + :width: 200px -A new statistical sampling profiler has been added to the :mod:`profile` module as -:mod:`profile.sample`. This profiler enables low-overhead performance analysis of +A new statistical sampling profiler (Tachyon) has been added as +:mod:`profiling.sampling`. This profiler enables low-overhead performance analysis of running Python processes without requiring code modification or process restart. -Unlike deterministic profilers (:mod:`cProfile` and :mod:`profile`) that instrument +Unlike deterministic profilers (such as :mod:`profiling.tracing`) that instrument every function call, the sampling profiler periodically captures stack traces from running processes. This approach provides virtually zero overhead while achieving sampling rates of **up to 1,000,000 Hz**, making it the fastest sampling profiler available for Python (at the time of its contribution) and ideal for debugging -performance issues in production environments. +performance issues in production environments. This capability is particularly +valuable for debugging performance issues in production systems where traditional +profiling approaches would be too intrusive. Key features include: * **Zero-overhead profiling**: Attach to any running Python process without - affecting its performance -* **No code modification required**: Profile existing applications without restart -* **Real-time statistics**: Monitor sampling quality during data collection -* **Multiple output formats**: Generate both detailed statistics and flamegraph data -* **Thread-aware profiling**: Option to profile all threads or just the main thread + affecting its performance. Ideal for production debugging where you can't afford + to restart or slow down your application. + +* **No code modification required**: Profile existing applications without restart. + Simply point the profiler at a running process by PID and start collecting data. + +* **Flexible target modes**: + + * Profile running processes by PID (``attach``) - attach to already-running applications + * Run and profile scripts directly (``run``) - profile from the very start of execution + * Execute and profile modules (``run -m``) - profile packages run as ``python -m module`` + * Capture a one-shot snapshot of a running process (``dump``) - print a + traceback-style stack of every thread (or all asyncio tasks with + ``--async-aware``). Useful for investigating hung processes. + +* **Multiple profiling modes**: Choose what to measure based on your performance investigation: + + * **Wall-clock time** (``--mode wall``, default): Measures real elapsed time including I/O, + network waits, and blocking operations. Use this to understand where your program spends + calendar time, including when waiting for external resources. + * **CPU time** (``--mode cpu``): Measures only active CPU execution time, excluding I/O waits + and blocking. Use this to identify CPU-bound bottlenecks and optimize computational work. + * **GIL-holding time** (``--mode gil``): Measures time spent holding Python's Global Interpreter + Lock. Use this to identify which threads dominate GIL usage in multi-threaded applications. + * **Exception handling time** (``--mode exception``): Captures samples only from threads with + an active exception. Use this to analyze exception handling overhead. + +* **Thread-aware profiling**: Option to profile all threads (``-a``) or just the main thread, + essential for understanding multi-threaded application behavior. + +* **Multiple output formats**: Choose the visualization that best fits your workflow: + + * ``--pstats``: Detailed tabular statistics compatible with :mod:`pstats`. Shows function-level + timing with direct and cumulative samples. Best for detailed analysis and integration with + existing Python profiling tools. + * ``--collapsed``: Generates collapsed stack traces (one line per stack). This format is + specifically designed for creating flame graphs with external tools like Brendan Gregg's + FlameGraph scripts or speedscope. + * ``--flamegraph``: Generates a self-contained interactive HTML flame graph using D3.js. + Opens directly in your browser for immediate visual analysis. Flame graphs show the call + hierarchy where width represents time spent, making it easy to spot bottlenecks at a glance. + * ``--gecko``: Generates Gecko Profiler format compatible with `Firefox Profiler + `__. Upload the output to Firefox Profiler for advanced + timeline-based analysis with features like stack charts, markers, and network activity. + * ``--heatmap``: Generates an interactive HTML heatmap visualization with line-level sample + counts. Creates a directory with per-file heatmaps showing exactly where time is spent + at the source code level. + +* **Live interactive mode**: Real-time TUI profiler with a top-like interface (``--live``). + Monitor performance as your application runs with interactive sorting and filtering. + +* **Async-aware profiling**: Profile async/await code with task-based stack reconstruction + (``--async-aware``). See which coroutines are consuming time, with options to show only + running tasks or all tasks including those waiting. + +* **Opcode-level profiling**: Gather bytecode opcode information for instruction-level + profiling (``--opcodes``). Shows which bytecode instructions are executing, including + specializations from the adaptive interpreter. + +See :mod:`profiling.sampling` for the complete documentation, including all +available output formats, profiling modes, and configuration options. + +(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`135953` and :gh:`138122`.) + + +.. _whatsnew315-frame-pointers: + +:pep:`831`: Frame pointers enabled by default +--------------------------------------------- + +CPython is now built with frame pointers by default on platforms that support +them. This uses the compiler flags ``-fno-omit-frame-pointer`` and +``-mno-omit-leaf-frame-pointer``, making native stack unwinding faster and +more reliable for system profilers, debuggers, crash analysis tools, and +eBPF-based observability tools. + +The flags are exposed through :mod:`sysconfig`, so extension modules built by +tools that consume Python's build configuration inherit frame pointers by +default. This propagation is intentional: mixed Python/native profiling needs +an unbroken frame-pointer chain through the interpreter, extension modules, +embedding applications, and native libraries. -Profile process 1234 for 10 seconds with default settings:: +.. important:: + + Third-party build backends and native build systems should preserve these + flags when they consume Python's :mod:`sysconfig` values. Build systems + that compile C, C++, Rust, or other native code without inheriting Python's + compiler flags should enable equivalent frame-pointer flags themselves. A + single native component built without frame pointers can break stack + unwinding for the whole Python process. + +(Contributed by Pablo Galindo Salgado and Savannah Ostrowski in +:gh:`149201`; PEP 831 written by Pablo Galindo Salgado, Ken Jin, +Savannah Ostrowski, and Diego Russo.) + +.. seealso:: :pep:`831` for further details. + + +.. _whatsnew315-unpacking-in-comprehensions: + +:pep:`798`: Unpacking in comprehensions +--------------------------------------- + +List, set, and dictionary comprehensions, as well as generator expressions, now +support unpacking with ``*`` and ``**``. This extends the unpacking syntax +from :pep:`448` to comprehensions, providing a new syntax for combining an +arbitrary number of iterables or dictionaries into a single flat structure. +This new syntax is a direct alternative to nested comprehensions, +:func:`itertools.chain`, and :meth:`itertools.chain.from_iterable`. For +example:: + + >>> lists = [[1, 2], [3, 4], [5]] + >>> [*L for L in lists] # equivalent to [x for L in lists for x in L] + [1, 2, 3, 4, 5] + + >>> sets = [{1, 2}, {2, 3}, {3, 4}] + >>> {*s for s in sets} # equivalent to {x for s in sets for x in s} + {1, 2, 3, 4} + + >>> dicts = [{'a': 1}, {'b': 2}, {'a': 3}] + >>> {**d for d in dicts} # equivalent to {k: v for d in dicts for k,v in d.items()} + {'a': 3, 'b': 2} + +Generator expressions can similarly use unpacking to yield values from multiple +iterables:: + + >>> gen = (*L for L in lists) # equivalent to (x for L in lists for x in L) + >>> list(gen) + [1, 2, 3, 4, 5] + +This change also extends to asynchronous generator expressions, such that, for +example, ``(*a async for a in agen())`` is equivalent to ``(x async for a in +agen() for x in a)``. + +.. seealso:: :pep:`798` for further details. + +(Contributed by Adam Hartz in :gh:`143055`.) + +.. _whatsnew315-startup-files: + +:pep:`829`: Package startup configuration files +----------------------------------------------- + +Loaded by the :mod:`site` module when ``-S`` is not given, :ref:`.pth files ` +can contain lines that both extend :data:`sys.path` and execute arbitrary code +when the line starts with ``import`` (followed by a space or tab). The latter +functionality can be problematic, since it is difficult to know exactly what +gets executed when Python starts up. - python -m profile.sample 1234 +As a step towards improving the ability to audit pre-start executable code, +Python 3.15 introduces :ref:`.start files ` which contain entry point +specifications of the form ``pkg.mod:callable`` where ``pkg.mod`` is the +import path to the given callable. When Python starts up, the callable is +located and called with no arguments. -Profile with custom interval and duration, save to file:: +``import`` lines in :file:`.pth` files are silently deprecated. When a +matching :file:`.start` file is found, ``import`` lines in :file:`.pth` files +are ignored. There is no change to :data:`sys.path` extension lines in +:file:`.pth` files. - python -m profile.sample -i 50 -d 30 -o profile.stats 1234 +The :mod:`site` module also provides :class:`site.StartupState` to batch +startup processing for multiple site directories, ensuring all static path +extensions are applied before any startup code is executed. :func:`site.main` +uses an instance of this class implicitly to batch process all startup +configuration files during normal interpreter startup. Callers needing the +same batching behavior can build a :class:`~site.StartupState` directly and +drive it with :meth:`~site.StartupState.addsitedir`, +:meth:`~site.StartupState.addusersitepackages`, and +:meth:`~site.StartupState.addsitepackages`, then call +:meth:`~site.StartupState.process` once at the end of the batch. -Generate collapsed stacks for flamegraph:: +(Contributed by Barry Warsaw in :gh:`148641` and :gh:`150228`.) - python -m profile.sample --collapsed 1234 -Profile all threads and sort by total time:: +.. _whatsnew315-abi3t: - python -m profile.sample -a --sort-tottime 1234 +:pep:`803` -- Stable ABI for free-threaded builds +------------------------------------------------- -The profiler generates statistical estimates of where time is spent:: +C extensions that target the :ref:`Stable ABI ` can now be +compiled for the new *Stable ABI for Free-Threaded Builds* (also known +as ``abi3t``), which makes them compatible with +:term:`free-threaded builds ` of CPython. +This usually requires some non-trivial changes to the source code; +specifically: - Real-time sampling stats: Mean: 100261.5Hz (9.97µs) Min: 86333.4Hz (11.58µs) Max: 118807.2Hz (8.42µs) Samples: 400001 - Captured 498841 samples in 5.00 seconds - Sample rate: 99768.04 samples/sec - Error rate: 0.72% - Profile Stats: - nsamples sample% tottime (s) cumul% cumtime (s) filename:lineno(function) - 43/418858 0.0 0.000 87.9 4.189 case.py:667(TestCase.run) - 3293/418812 0.7 0.033 87.9 4.188 case.py:613(TestCase._callTestMethod) - 158562/158562 33.3 1.586 33.3 1.586 test_compile.py:725(TestSpecifics.test_compiler_recursion_limit..check_limit) - 129553/129553 27.2 1.296 27.2 1.296 ast.py:46(parse) - 0/128129 0.0 0.000 26.9 1.281 test_ast.py:884(AST_Tests.test_ast_recursion_limit..check_limit) - 7/67446 0.0 0.000 14.2 0.674 test_compile.py:729(TestSpecifics.test_compiler_recursion_limit) - 6/60380 0.0 0.000 12.7 0.604 test_ast.py:888(AST_Tests.test_ast_recursion_limit) - 3/50020 0.0 0.000 10.5 0.500 test_compile.py:727(TestSpecifics.test_compiler_recursion_limit) - 1/38011 0.0 0.000 8.0 0.380 test_ast.py:886(AST_Tests.test_ast_recursion_limit) - 1/25076 0.0 0.000 5.3 0.251 test_compile.py:728(TestSpecifics.test_compiler_recursion_limit) - 22361/22362 4.7 0.224 4.7 0.224 test_compile.py:1368(TestSpecifics.test_big_dict_literal) - 4/18008 0.0 0.000 3.8 0.180 test_ast.py:889(AST_Tests.test_ast_recursion_limit) - 11/17696 0.0 0.000 3.7 0.177 subprocess.py:1038(Popen.__init__) - 16968/16968 3.6 0.170 3.6 0.170 subprocess.py:1900(Popen._execute_child) - 2/16941 0.0 0.000 3.6 0.169 test_compile.py:730(TestSpecifics.test_compiler_recursion_limit) +- Switching to API introduced in :pep:`697` (Python 3.12), such as + negative :c:member:`~PyType_Spec.basicsize` and + :c:func:`PyObject_GetTypeData`, rather than making :c:type:`PyObject` + part of the instance struct; and +- Switching from a ``PyInit_`` function to a new export hook, + :c:func:`PyModExport_* `, introduced for this + purpose in :pep:`793`, with a new :c:type:`PySlot` structure + introduced in :pep:`820`. - Legend: - nsamples: Direct/Cumulative samples (direct executing / on call stack) - sample%: Percentage of total samples this function was directly executing - tottime: Estimated total time spent directly in this function - cumul%: Percentage of total samples when this function was on the call stack - cumtime: Estimated cumulative time (including time in called functions) - filename:lineno(function): Function location and name +Note that Stable ABI does not offer all the functionality that CPython +has to offer. +Extensions that cannot switch to ``abi3t`` should continue to build for +the existing Stable ABI (``abi3``) and the version-specific ABI for +free-threading (``cp315t``) separately. - Summary of Interesting Functions: +Stable ABI for Free-Threaded Builds should typically +be selected in a build tool (such as, for example, Setuptools, meson-python, +scikit-build-core, or Maturin). +At the time of writing, these tools did **not** support ``abi3t``. +If this is the case for your tool, compile for ``cp315t`` separately. +If not using a build tool -- or when writing such a tool -- you can select +``abi3t`` by setting the macro :c:macro:`!Py_TARGET_ABI3T` as discussed +in :ref:`abi3-compiling`. - Functions with Highest Direct/Cumulative Ratio (Hot Spots): - 1.000 direct/cumulative ratio, 33.3% direct samples: test_compile.py:(TestSpecifics.test_compiler_recursion_limit..check_limit) - 1.000 direct/cumulative ratio, 27.2% direct samples: ast.py:(parse) - 1.000 direct/cumulative ratio, 3.6% direct samples: subprocess.py:(Popen._execute_child) +A practical :ref:`migration guide ` for switching to +``abi3t`` is available. - Functions with Highest Call Frequency (Indirect Calls): - 418815 indirect calls, 87.9% total stack presence: case.py:(TestCase.run) - 415519 indirect calls, 87.9% total stack presence: case.py:(TestCase._callTestMethod) - 159470 indirect calls, 33.5% total stack presence: test_compile.py:(TestSpecifics.test_compiler_recursion_limit) +.. seealso:: :pep:`803` for further details. - Functions with Highest Call Magnification (Cumulative/Direct): - 12267.9x call magnification, 159470 indirect calls from 13 direct: test_compile.py:(TestSpecifics.test_compiler_recursion_limit) - 10581.7x call magnification, 116388 indirect calls from 11 direct: test_ast.py:(AST_Tests.test_ast_recursion_limit) - 9740.9x call magnification, 418815 indirect calls from 43 direct: case.py:(TestCase.run) -The profiler automatically identifies performance bottlenecks through statistical -analysis, highlighting functions with high CPU usage and call frequency patterns. +.. _whatsnew315-c-api-interpreter-finalization: -This capability is particularly valuable for debugging performance issues in -production systems where traditional profiling approaches would be too intrusive. +:pep:`788`: Protecting the C API from interpreter finalization +-------------------------------------------------------------- -(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`135953`.) +In the C API, :term:`interpreter finalization ` can be +problematic for many extensions, because :term:`attaching ` a thread state will permanently hang the thread, resulting in deadlocks +and other spurious issues. Additionally, it has historically been impossible +to safely check whether an interpreter is alive before using it, leading to crashes +when a thread concurrently deletes an interpreter while another thread is +trying to attach to it. + +There are now several new suites of APIs to circumvent these problems: + +* :ref:`Interpreter guards `, which prevent an interpreter + from finalizing. +* :ref:`Interpreter views `, which allow thread-safe access + to an interpreter that may be concurrently finalizing or deleted. +* :ref:`New APIs ` to automatically attach and detach + thread states that come with built-in protection against finalization. + +In addition, APIs in the ``PyGILState`` family (most notably +:c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release`) have been +:term:`soft deprecated`. There is **no** plan to remove them, and existing +code will continue to work, but there will be no new ``PyGILState`` APIs +in future versions of Python. + +.. seealso:: :pep:`788` for further details. + +(Contributed by Peter Bierma in :gh:`149101`.) + + +.. _whatsnew315-improved-error-messages: + +Improved error messages +----------------------- + +* The interpreter now provides more helpful suggestions in :exc:`AttributeError` + exceptions when accessing an attribute on an object that does not exist, but + a similar attribute is available through one of its members. + + For example, if the object has an attribute that itself exposes the requested + name, the error message will suggest accessing it via that inner attribute: + + .. code-block:: python + + @dataclass + class Circle: + radius: float + + @property + def area(self) -> float: + return pi * self.radius**2 + + class Container: + def __init__(self, inner: Circle) -> None: + self.inner = inner + + circle = Circle(radius=4.0) + container = Container(circle) + print(container.area) + + Running this code now produces a clearer suggestion: + + .. code-block:: pytb + + Traceback (most recent call last): + File "/home/pablogsal/github/python/main/lel.py", line 42, in + print(container.area) + ^^^^^^^^^^^^^^ + AttributeError: 'Container' object has no attribute 'area'. Did you mean '.inner.area' instead of '.area'? + +* When an :exc:`AttributeError` on a builtin type has no close match via + Levenshtein distance, the error message now checks a static table of common + method names from other languages (JavaScript, Java, Ruby, C#) and suggests + the Python equivalent: + + .. doctest:: + + >>> [1, 2, 3].push(4) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'list' object has no attribute 'push'. Did you mean '.append'? + + >>> 'hello'.toUpperCase() # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'str' object has no attribute 'toUpperCase'. Did you mean '.upper'? + + When the Python equivalent is a language construct rather than a method, + the hint describes the construct directly: + + .. doctest:: + + >>> {}.put("a", 1) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'dict' object has no attribute 'put'. Use d[k] = v. + + When a mutable method is called on an immutable type, the hint suggests + the mutable counterpart: + + .. doctest:: + + >>> (1, 2, 3).append(4) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'tuple' object has no attribute 'append'. Did you mean to use a 'list' object? + + These hints also work for subclasses of builtin types. + + (Contributed by Matt Van Horn in :gh:`146406`.) + +* The interpreter now tries to provide a suggestion when + :func:`delattr` fails due to a missing attribute. + When an attribute name that closely resembles an existing attribute is used, + the interpreter will suggest the correct attribute name in the error message. + For example: + + .. doctest:: + + >>> class A: + ... pass + >>> a = A() + >>> a.abcde = 1 + >>> del a.abcdf # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'A' object has no attribute 'abcdf'. Did you mean: 'abcde'? + + (Contributed by Nikita Sobolev and Pranjal Prajapati in :gh:`136588`.) + +* Several error messages incorrectly using the term "argument" have been corrected. + (Contributed by Stan Ulbrych in :gh:`133382`.) Other language changes ====================== +.. _whatsnew315-utf8-default: + * Python now uses UTF-8_ as the default encoding, independent of the system's environment. This means that I/O operations without an explicit encoding, - e.g. ``open('flying-circus.txt')``, will use UTF-8. + for example, ``open('flying-circus.txt')``, will use UTF-8. UTF-8 is a widely-supported Unicode_ character encoding that has become a *de facto* standard for representing text, including nearly every webpage on the internet, many common file formats, programming languages, and more. @@ -183,12 +687,12 @@ Other language changes compatibility between versions of Python, ensure that an explicit ``encoding`` argument is always provided. The :ref:`opt-in encoding warning ` can be used to identify code that may be affected by this change. - The special special ``encoding='locale'`` argument uses the current locale + The special ``encoding='locale'`` argument uses the current locale encoding, and has been supported since Python 3.10. To retain the previous behaviour, Python's UTF-8 mode may be disabled with the :envvar:`PYTHONUTF8=0 ` environment variable or the - :option:`-X utf8=0 <-X>` command line option. + :option:`-X utf8=0 <-X>` command-line option. .. seealso:: :pep:`686` for further details. @@ -197,136 +701,1161 @@ Other language changes (Contributed by Adam Turner in :gh:`133711`; PEP 686 written by Inada Naoki.) -* Several error messages incorrectly using the term "argument" have been corrected. - (Contributed by Stan Ulbrych in :gh:`133382`.) + .. _whatsnew315-color-interpreter-help: + +* The interpreter help (such as ``python --help``) is now in color. + This can be controlled by :ref:`environment variables + `. + (Contributed by Hugo van Kemenade in :gh:`148766`.) + + .. _whatsnew315-color-exceptions: + +* Unraisable exceptions are now highlighted with color by default. This can be + controlled by :ref:`environment variables `. + (Contributed by Peter Bierma in :gh:`134170`.) + + .. _whatsnew315-more-color: + +* More color in + :ref:`argparse `, + :ref:`ast `, + :ref:`calendar `, + :ref:`difflib `, + :ref:`http.server `, + :ref:`pickletools `, + :ref:`PyREPL tab completion `, + :ref:`python --help `, + :ref:`sqlite3 `, + :ref:`timeit `, + :ref:`tokenize `, + :ref:`unraisable exceptions ` and + :term:`stdlib` (ast, compileall, doctest, gzip, inspect, json.tool, pdb, + profiling.sampling, random, regrtest, sqlite3, timeit, tokenize, trace, + unittest, uuid, zipapp, zipfile) CLI help. + +* The :meth:`~object.__repr__` of :class:`ImportError` and :class:`ModuleNotFoundError` + now shows "name" and "path" as ``name=`` and ``path=`` if they were given + as keyword arguments at construction time. + (Contributed by Serhiy Storchaka, Oleg Iarygin, and Yoav Nir in :gh:`74185`.) + +* The :attr:`~object.__dict__` and :attr:`!__weakref__` descriptors now use a + single descriptor instance per interpreter, shared across all types that + need them. + This speeds up class creation, and helps avoid reference cycles. + (Contributed by Petr Viktorin in :gh:`135228`.) + +* The :option:`-W` option and the :envvar:`PYTHONWARNINGS` environment variable + can now specify regular expressions instead of literal strings to match + the warning message and the module name, if the corresponding field starts + and ends with a forward slash (``/``). + (Contributed by Serhiy Storchaka in :gh:`134716`.) + +* Functions that take timestamp or timeout arguments now accept any real + numbers (such as :class:`~decimal.Decimal` and :class:`~fractions.Fraction`), + not only integers or floats, although this does not improve precision. + (Contributed by Serhiy Storchaka in :gh:`67795`.) + +.. _whatsnew315-bytearray-take-bytes: + +* Added :meth:`bytearray.take_bytes(n=None, /) ` to take + bytes out of a :class:`bytearray` without copying. This enables optimizing code + which must return :class:`bytes` after working with a mutable buffer of bytes + such as data buffering, network protocol parsing, encoding, decoding, + and compression. Common code patterns which can be optimized with + :func:`~bytearray.take_bytes` are listed below. + + .. list-table:: Suggested optimizing refactors + :header-rows: 1 + + * - Description + - Old + - New + + * - Return :class:`bytes` after working with :class:`bytearray` + - .. code:: python + + def read() -> bytes: + buffer = bytearray(1024) + ... + return bytes(buffer) + + - .. code:: python + + def read() -> bytes: + buffer = bytearray(1024) + ... + return buffer.take_bytes() + + * - Empty a buffer getting the bytes + - .. code:: python + + buffer = bytearray(1024) + ... + data = bytes(buffer) + buffer.clear() + + - .. code:: python + + buffer = bytearray(1024) + ... + data = buffer.take_bytes() + + * - Split a buffer at a specific separator + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + data = bytes(buffer[:n + 1]) + del buffer[:n + 1] + assert data == b'abc\n' + assert buffer == bytearray(b'def') + + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + data = buffer.take_bytes(n + 1) + + * - Split a buffer at a specific separator; discard after the separator + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + data = bytes(buffer[:n]) + buffer.clear() + assert data == b'abc' + assert len(buffer) == 0 + + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + buffer.resize(n) + data = buffer.take_bytes() + + (Contributed by Cody Maloney in :gh:`139871`.) + +* Many functions related to compiling or parsing Python code, such as + :func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, + and :func:`importlib.abc.InspectLoader.source_to_code`, now allow the module + name to be passed. It is needed to unambiguously :ref:`filter ` + syntax warnings by module name. + (Contributed by Serhiy Storchaka in :gh:`135801`.) + +* Allowed defining the *__dict__* and *__weakref__* :ref:`__slots__ ` + for any class. + (Contributed by Serhiy Storchaka in :gh:`41779`.) + +* Allowed defining any :ref:`__slots__ ` for a class derived from + :class:`tuple` (including classes created by :func:`collections.namedtuple`). + (Contributed by Serhiy Storchaka in :gh:`41779`.) + +* The :class:`slice` type now supports subscription, + making it a :term:`generic type`. + (Contributed by James Hilton-Balfe in :gh:`128335`.) + +* The class :class:`memoryview` now supports the :c:expr:`float complex` and + :c:expr:`double complex` C types: formatting characters ``'Zf'`` and ``'Zd'`` + respectively. + (Contributed by Victor Stinner in :gh:`146151` and :gh:`148675`.) + +* Allow the *count* argument of :meth:`bytes.replace` to be a keyword. + (Contributed by Stan Ulbrych in :gh:`147856`.) + +* Unary plus is now accepted in :keyword:`match` literal patterns, mirroring + the existing support for unary minus. + (Contributed by Bartosz Sławecki in :gh:`145239`.) + +* The import system now acquires per-module locks in hierarchical order + (parent packages before their submodules). This fixes a long-standing + deadlock where one thread importing ``pkg.sub`` and another importing + ``pkg.sub.mod`` could each block the other when ``pkg/sub/__init__.py`` + imports ``pkg.sub.mod``. + (Contributed by Gregory P. Smith in :gh:`83065`.) + + +Default interactive shell +========================= + +.. _whatsnew315-pyrepl-completion: + +* Tab completions are now colored by object kind, based on + :pypi:`fancycompleter`. + Set :envvar:`PYTHON_BASIC_COMPLETER` to fall back to :mod:`rlcompleter`. + Color can also be controlled by :ref:`environment variables + `. + (Contributed by Antonio Cuni and Pablo Galindo in :gh:`130472`.) New modules =========== -* None yet. +math.integer +------------ + +This module provides access to the mathematical functions for integer +arguments (:pep:`791`). +(Contributed by Serhiy Storchaka in :gh:`81313`.) Improved modules ================ +argparse +-------- + +* The :class:`~argparse.BooleanOptionalAction` action now supports single-dash + long options and alternate prefix characters. + (Contributed by Serhiy Storchaka in :gh:`138525`.) + +* Changed the *suggest_on_error* parameter of :class:`argparse.ArgumentParser` to + default to ``True``. This enables suggestions for mistyped arguments by default. + (Contributed by Jakob Schluse in :gh:`140450`.) + + .. _whatsnew315-color-argparse: + +* Added backtick markup support in :class:`~argparse.ArgumentParser` description + and epilog text to highlight inline code when color output is enabled. + (Contributed by Savannah Ostrowski in :gh:`142390`.) + +* Extended backtick markup to argument ``help`` text and added support for + double backticks (RST inline-literal style). + (Contributed by Hugo van Kemenade in :gh:`149375`.) + + +array +----- + +* Support the :c:expr:`float complex` and :c:expr:`double complex` C types: + formatting characters ``'Zf'`` and ``'Zd'`` respectively. + (Contributed by Victor Stinner in :gh:`146151` and :gh:`148675`.) + +* Support half-floats (16-bit IEEE 754 binary interchange format): formatting + character ``'e'``. + (Contributed by Sergey B Kirpichev in :gh:`146238`.) + +* The :data:`array.typecodes` type changed from :class:`str` to :class:`tuple` + to support type codes longer than 1 character (``Zf`` and ``Zd``). + (Contributed by Victor Stinner in :gh:`148675`.) + + +ast +--- + + .. _whatsnew315-color-ast: + +* Add *color* parameter to :func:`~ast.dump`. + If ``True``, the returned string is syntax highlighted using ANSI escape + sequences. + If ``False`` (the default), colored output is always disabled. + (Contributed by Stan Ulbrych in :gh:`148981`.) + +* The :ref:`command-line ` output is now syntax highlighted by default. + This can be :ref:`controlled using environment variables `. + (Contributed by Stan Ulbrych in :gh:`148981`.) + + +asyncio +------- + +* Added :meth:`TaskGroup.cancel ` to allow early + termination of a task group, for instance, when the goal of the tasks has + been achieved or their services are no longer needed. + Previously this would involve unintuitive boilerplate such as an extra task + raising a custom exception which is then suppressed as it exits the task group. + (Contributed by John Belmonte in :gh:`127214`.) + + +base64 +------ + +* Added the *pad* parameter in :func:`~base64.z85encode`. + (Contributed by Hauke Dämpfling in :gh:`143103`.) + +* Added the *padded* parameter in + :func:`~base64.b32encode`, :func:`~base64.b32decode`, + :func:`~base64.b32hexencode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b64encode`, :func:`~base64.b64decode`, + :func:`~base64.urlsafe_b64encode`, and :func:`~base64.urlsafe_b64decode`. + (Contributed by Serhiy Storchaka in :gh:`73613`.) + +* Added the *wrapcol* parameter in :func:`~base64.b16encode`, + :func:`~base64.b32encode`, :func:`~base64.b32hexencode`, + :func:`~base64.b64encode`, :func:`~base64.b85encode`, and + :func:`~base64.z85encode`. + (Contributed by Serhiy Storchaka in :gh:`143214` and :gh:`146431`.) + +* Added the *ignorechars* parameter in :func:`~base64.b16decode`, + :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b64decode`, :func:`~base64.b85decode`, and + :func:`~base64.z85decode`. + (Contributed by Serhiy Storchaka in :gh:`144001` and :gh:`146431`.) + +* Added the *canonical* parameter in + :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b64decode`, :func:`~base64.urlsafe_b64decode`, + :func:`~base64.a85decode`, :func:`~base64.b85decode`, and + :func:`~base64.z85decode`, + to reject encodings with non-zero padding bits or other non-canonical + forms. + (Contributed by Gregory P. Smith in :gh:`146311`.) + + +binascii +-------- + +* Added functions for Base32 encoding: + + - :func:`~binascii.b2a_base32` and :func:`~binascii.a2b_base32` + + (Contributed by James Seo in :gh:`146192`.) + +* Added functions for Ascii85, Base85, and Z85 encoding: + + - :func:`~binascii.b2a_ascii85` and :func:`~binascii.a2b_ascii85` + - :func:`~binascii.b2a_base85` and :func:`~binascii.a2b_base85` + + (Contributed by James Seo and Serhiy Storchaka in :gh:`101178`.) + +* Added the *padded* parameter in + :func:`~binascii.b2a_base32`, :func:`~binascii.a2b_base32`, + :func:`~binascii.b2a_base64`, and :func:`~binascii.a2b_base64`. + (Contributed by Serhiy Storchaka in :gh:`73613`.) + +* Added the *wrapcol* parameter in :func:`~binascii.b2a_base64`. + (Contributed by Serhiy Storchaka in :gh:`143214`.) + +* Added the *alphabet* parameter in :func:`~binascii.b2a_base64` and + :func:`~binascii.a2b_base64`. + (Contributed by Serhiy Storchaka in :gh:`145980`.) + +* Added the *ignorechars* parameter in :func:`~binascii.a2b_hex`, + :func:`~binascii.unhexlify`, and :func:`~binascii.a2b_base64`. + (Contributed by Serhiy Storchaka in :gh:`144001` and :gh:`146431`.) + +* Added the *canonical* parameter in :func:`~binascii.a2b_base64`, + to reject encodings with non-zero padding bits. + (Contributed by Gregory P. Smith in :gh:`146311`.) + + +calendar +-------- + + .. _whatsnew315-color-calendar: + +* :mod:`calendar`'s :ref:`command-line ` text output has more + color. This can be controlled with :ref:`environment variables + `. + (Contributed by Hugo van Kemenade in :gh:`148352`.) + +* The :mod:`calendar`'s :ref:`command-line ` HTML output now + accepts the year-month option: ``python -m calendar -t html 2009 06``. + (Contributed by Pål Grønås Drange in :gh:`140212`.) + +* Calendar pages generated by the :class:`calendar.HTMLCalendar` class now support + dark mode and have been migrated to the HTML5 standard for improved accessibility. + (Contributed by Jiahao Li and Hugo van Kemenade in :gh:`137634`.) + + +collections +----------- + +* Added :meth:`!collections.Counter.__xor__` and + :meth:`!collections.Counter.__ixor__` to compute the symmetric difference + between :class:`~collections.Counter` objects. + (Contributed by Raymond Hettinger in :gh:`138682`.) + + +concurrent.futures +------------------ + +* Improved error reporting when a child process in a + :class:`concurrent.futures.ProcessPoolExecutor` terminates abruptly. + The resulting traceback will now tell you the PID and exit code of the + terminated process. + (Contributed by Jonathan Berg in :gh:`139486`.) + + +contextlib +---------- + +* Added support for arbitrary descriptors :meth:`!__enter__`, + :meth:`!__exit__`, :meth:`!__aenter__`, and :meth:`!__aexit__` in + :class:`~contextlib.ExitStack` and :class:`contextlib.AsyncExitStack`, for + consistency with the :keyword:`with` and :keyword:`async with` statements. + (Contributed by Serhiy Storchaka in :gh:`144386`.) + +* :class:`~contextlib.ContextDecorator` and + :class:`~contextlib.AsyncContextDecorator` (and therefore + :func:`~contextlib.contextmanager` and :func:`~contextlib.asynccontextmanager` + used as decorators) now detect generator functions, coroutine functions, and + asynchronous generator functions and keep the context manager open across + iteration or await. Previously the context manager exited as soon as the + generator or coroutine object was created. + (Contributed by Alex Grönholm & Gregory P. Smith in :gh:`125862`.) + + +dataclasses +----------- + +* Annotations for generated ``__init__`` methods no longer include internal + type names. + + dbm --- * Added new :meth:`!reorganize` methods to :mod:`dbm.dumb` and :mod:`dbm.sqlite3` - which allow to recover unused free space previously occupied by deleted entries. + to recover unused free space previously occupied by deleted entries. + (Contributed by Andrea Oliveri in :gh:`134004`.) + + +difflib +------- + + .. _whatsnew315-color-difflib: + +* Introduced the optional *color* parameter to :func:`difflib.unified_diff`, + enabling color output similar to :program:`git diff`. + This can be controlled by :ref:`environment variables + `. + (Contributed by Douglas Thor in :gh:`133725`.) + +* Improved the styling of HTML diff pages generated by the :class:`difflib.HtmlDiff` + class, and migrated the output to the HTML5 standard. + (Contributed by Jiahao Li in :gh:`134580`.) + + +email +----- + +* Email generators now raise an error when an :class:`.EmailMessage` cannot be + accurately flattened due to a non-ASCII email address (mailbox) in an address + header. Options for supporting Email Address Internationalization (EAI) are + discussed in :attr:`.EmailPolicy.utf8`. + (Contributed by R David Murray and Mike Edmunds in :gh:`122540`.) + + +faulthandler +------------ + +* Added the *max_threads* parameter in :func:`faulthandler.enable`, + :func:`faulthandler.dump_traceback`, :func:`faulthandler.dump_traceback_later`, + and :func:`faulthandler.register`. + (Contributed by Eric Froemling in :gh:`149085`.) + + +functools +--------- + +* :func:`~functools.singledispatchmethod` now supports non-:term:`descriptor` + callables. + (Contributed by Serhiy Storchaka in :gh:`140873`.) + +* :func:`~functools.singledispatchmethod` now dispatches on the second argument + if it wraps a regular method and is called as a class attribute. + (Contributed by Bartosz Sławecki in :gh:`143535`.) + + +gc +-- + +* Python 3.14.0-3.14.4 shipped with a new incremental garbage collector. + However, due to a number of `reports + `__ + of significant memory pressure in production environments, + it has been reverted back to the generational GC from 3.13. + This is the GC now used in Python 3.14.5 and later and Python 3.15. + + +hashlib +------- + +* Ensure that hash functions guaranteed to be always *available* exist as + attributes of :mod:`hashlib` even if they will not work at runtime due to + missing backend implementations. For instance, ``hashlib.md5`` will no + longer raise :exc:`AttributeError` if OpenSSL is not available and Python + has been built without MD5 support. + (Contributed by Bénédikt Tran in :gh:`136929`.) + + +http.client +----------- + +* A new *max_response_headers* keyword-only parameter has been added to + :class:`~http.client.HTTPConnection` and :class:`~http.client.HTTPSConnection` + constructors. This parameter overrides the default maximum number of allowed + response headers. + (Contributed by Alexander Enrique Urieles Nieto in :gh:`131724`.) + + +http.server +----------- + + .. _whatsnew315-color-http.server: + +* The logging of :mod:`~http.server.BaseHTTPRequestHandler`, + as used by the :ref:`command-line interface `, + is colored by default. + This can be controlled with :ref:`environment variables + `. + (Contributed by Hugo van Kemenade in :gh:`146292`.) + +* Added :attr:`~http.server.SimpleHTTPRequestHandler.default_content_type` + and the :option:`--content-type ` command-line + option to allow customizing the default ``Content-Type`` header + for files with unknown extensions. + (Contributed by John Comeau and Hugo van Kemenade in :gh:`113471`.) + +* Add a new ``extra_response_headers`` keyword argument to + :class:`~http.server.SimpleHTTPRequestHandler` to support custom headers in + HTTP responses. + (Contributed by Anton I. Sipos in :gh:`135057`.) + +* Add a ``-H/--header`` option to the :program:`python -m http.server` + command-line interface to support custom headers in HTTP responses. + (Contributed by Anton I. Sipos in :gh:`135057`.) + + +importlib.metadata +------------------ + +* Previously, when accessing a distribution metadata directory not + containing a metadata file, ``metadata()`` and ``Distribution.metadata()`` + would return an empty ``PackageMetadata`` object as if the file + was present but empty. Now, a ``MetadataNotFound`` exception is raised. + See `importlib_metadata#493 `_ + for background and rationale and and :gh:`143387` for rationale on the + compatibility concerns. + (Contributed by Jason R. Coombs.) + + +inspect +------- + +* Add parameters *inherit_class_doc* and *fallback_to_class_doc* + for :func:`~inspect.getdoc`. + (Contributed by Serhiy Storchaka in :gh:`132686`.) + + +json +---- + +* Add the *array_hook* parameter to :func:`~json.load` and + :func:`~json.loads` functions: + allow a callback for JSON literal array types to customize Python lists in + the resulting decoded object. Passing combined :class:`frozendict` to + *object_pairs_hook* param and :class:`tuple` to ``array_hook`` will yield a + deeply nested immutable Python structure representing the JSON data. + (Contributed by Joao S. O. Bueno in :gh:`146440`.) + + +locale +------ + +* :func:`~locale.setlocale` now supports language codes with ``@``-modifiers. + ``@``-modifiers are no longer silently removed in :func:`~locale.getlocale`, + but included in the language code. + (Contributed by Serhiy Storchaka in :gh:`137729`.) + +* Undeprecate the :func:`locale.getdefaultlocale` function. + (Contributed by Victor Stinner in :gh:`130796`.) + + +math +---- + +* Add :func:`math.isnormal` and :func:`math.issubnormal` functions. + (Contributed by Sergey B Kirpichev in :gh:`132908`.) + +* Add :func:`math.fmax`, :func:`math.fmin` and :func:`math.signbit` functions. + (Contributed by Bénédikt Tran in :gh:`135853`.) + + +mimetypes +--------- + +* Add more MIME types. + (Contributed by Benedikt Johannes, Charlie Lin, Foolbar, Gil Forcada and + John Franey + in :gh:`144217`, :gh:`145720`, :gh:`140937`, :gh:`139959`, :gh:`145698`, + :gh:`145718`, :gh:`145918`, and :gh:`144213`.) +* Rename ``application/x-texinfo`` to ``application/texinfo``. + (Contributed by Charlie Lin in :gh:`140165`.) +* Changed the MIME type for ``.ai`` files to ``application/pdf``. + (Contributed by Stan Ulbrych in :gh:`141239`.) + + +mmap +---- + +* :class:`mmap.mmap` now has a *trackfd* parameter on Windows; + if it is ``False``, the file handle corresponding to *fileno* will + not be duplicated. + (Contributed by Serhiy Storchaka in :gh:`78502`.) + +* Added the :meth:`mmap.mmap.set_name` method + to annotate an anonymous memory mapping + if Linux kernel supports :manpage:`PR_SET_VMA_ANON_NAME ` (Linux 5.17 or newer). + (Contributed by Donghee Na in :gh:`142419`.) + + +os +-- + +* Add :func:`os.statx` on Linux kernel versions 4.11 and later with + glibc versions 2.28 and later. + (Contributed by Jeffrey Bosboom and Victor Stinner in :gh:`83714`.) + +* :func:`os.makedirs` function now has a *parent_mode* parameter that allows + specifying the mode for intermediate directories. This can be used to match + the behavior from Python 3.6 and earlier by passing ``parent_mode=mode``. + (Contributed by Zackery Spytz and Gregory P. Smith in :gh:`86533`.) + +os.path +------- + +* Add support of the all-but-last mode in :func:`~os.path.realpath`. + (Contributed by Serhiy Storchaka in :gh:`71189`.) + +* The *strict* parameter to :func:`os.path.realpath` accepts a new value, + :data:`os.path.ALLOW_MISSING`. + If used, errors other than :exc:`FileNotFoundError` will be re-raised; + the resulting path can be missing but it will be free of symlinks. + (Contributed by Petr Viktorin for :cve:`2025-4517`.) + + +pdb +--- + +* Use the new interactive shell as the default input shell for :mod:`pdb`. + (Contributed by Tian Gao in :gh:`145379`.) + + +pickle +------ + +* Add support for pickling private methods and nested classes. + (Contributed by Zackery Spytz and Serhiy Storchaka in :gh:`77188`.) + + +pickletools +----------- + + .. _whatsnew315-color-pickletools: + +* The output of the :mod:`pickletools` command-line interface is colored by + default. This can be controlled with + :ref:`environment variables `. + (Contributed by Hugo van Kemenade in :gh:`149026`.) + + +pprint +------ + +* Add an *expand* keyword argument for :func:`pprint.pprint`, + :func:`pprint.pformat`, :func:`pprint.pp`. If true, the output will be + formatted similar to pretty-printed :func:`json.dumps` when + *indent* is supplied. + (Contributed by Stefan Todoran, Semyon Moroz and Hugo van Kemenade in + :gh:`112632`.) + +* Add t-string support to :mod:`pprint`. + (Contributed by Loïc Simon and Hugo van Kemenade in :gh:`134551`.) + + +re +-- + +* :func:`re.prefixmatch` and a corresponding :meth:`re.Pattern.prefixmatch` + have been added as alternate, more explicit names for the existing + and now :term:`soft deprecated` + :func:`re.match` and :meth:`re.Pattern.match` APIs. These are intended + to be used to alleviate confusion around what *match* means by following the + Zen of Python's *"Explicit is better than implicit"* mantra. Most other + language regular expression libraries use an API named *match* to mean what + Python has always called *search*. + (Contributed by Gregory P. Smith in :gh:`86519`.) + + +resource +-------- + +* Add new constants: :data:`~resource.RLIMIT_NTHR`, + :data:`~resource.RLIMIT_UMTXP`, :data:`~resource.RLIMIT_THREADS`, + :data:`~resource.RLIM_SAVED_CUR`, and :data:`~resource.RLIM_SAVED_MAX`. + (Contributed by Serhiy Storchaka in :gh:`137512`.) + + +shelve +------ + +* Added new :meth:`!reorganize` method to :mod:`shelve` used to recover unused free + space previously occupied by deleted entries. (Contributed by Andrea Oliveri in :gh:`134004`.) +* Add support for custom serialization and deserialization functions + in the :mod:`shelve` module. + (Contributed by Furkan Onder in :gh:`99631`.) + + +socket +------ + +* Add constants for the ISO-TP CAN protocol. + (Contributed by Patrick Menschel and Stefan Tatschner in :gh:`86819`.) + + +sqlite3 +------- + +* The :ref:`command-line interface ` has several new features: + + * SQL keyword completion on . + (Contributed by Long Tan in :gh:`133393`.) + + .. _whatsnew315-color-sqlite3: + + * Prompts, error messages, and help text are now colored. + This is enabled by default, see :ref:`using-on-controlling-color` for + details. + (Contributed by Stan Ulbrych and Łukasz Langa in :gh:`133461`.) + + * Table, index, trigger, view, column, function, and schema completion on . + (Contributed by Long Tan in :gh:`136101`.) + + +ssl +--- + +* Indicate through :data:`ssl.HAS_PSK_TLS13` whether the :mod:`ssl` module + supports "External PSKs" in TLSv1.3, as described in :rfc:`9258`. + (Contributed by Will Childs-Klein in :gh:`133624`.) + +* Added new methods for managing groups used for SSL key agreement + + * :meth:`ssl.SSLContext.set_groups` sets the groups allowed for doing + key agreement, extending the previous + :meth:`ssl.SSLContext.set_ecdh_curve` method. + This new API provides the ability to list multiple groups and + supports fixed-field and post-quantum groups in addition to ECDH + curves. This method can also be used to control what key shares + are sent in the TLS handshake. + * :meth:`ssl.SSLSocket.group` returns the group selected for doing key + agreement on the current connection after the TLS handshake completes. + This call requires OpenSSL 3.2 or later. + * :meth:`ssl.SSLContext.get_groups` returns a list of all available key + agreement groups compatible with the minimum and maximum TLS versions + currently set in the context. This call requires OpenSSL 3.5 or later. + + (Contributed by Ron Frederick in :gh:`136306`.) + +* Added a new method :meth:`ssl.SSLContext.set_ciphersuites` for setting TLS 1.3 + ciphers. For TLS 1.2 or earlier, :meth:`ssl.SSLContext.set_ciphers` should + continue to be used. Both calls can be made on the same context and the + selected cipher suite will depend on the TLS version negotiated when a + connection is made. + (Contributed by Ron Frederick in :gh:`137197`.) + +* Added new methods for managing signature algorithms: + + * :func:`ssl.get_sigalgs` returns a list of all available TLS signature + algorithms. This call requires OpenSSL 3.4 or later. + * :meth:`ssl.SSLContext.set_client_sigalgs` sets the signature algorithms + allowed for certificate-based client authentication. + * :meth:`ssl.SSLContext.set_server_sigalgs` sets the signature algorithms + allowed for the server to complete the TLS handshake. + * :meth:`ssl.SSLSocket.client_sigalg` returns the signature algorithm + selected for client authentication on the current connection. This call + requires OpenSSL 3.5 or later. + * :meth:`ssl.SSLSocket.server_sigalg` returns the signature algorithm + selected for the server to complete the TLS handshake on the current + connection. This call requires OpenSSL 3.5 or later. + + (Contributed by Ron Frederick in :gh:`138252`.) + + +subprocess +---------- + +* :meth:`subprocess.Popen.wait`: when ``timeout`` is not ``None`` and the + platform supports it, an efficient event-driven mechanism is used to wait for + process termination: + + - Linux >= 5.3 uses :func:`os.pidfd_open` + :func:`select.poll`. + - macOS and other BSD variants use :func:`select.kqueue` + ``KQ_FILTER_PROC`` + ``KQ_NOTE_EXIT``. + - Windows keeps using ``WaitForSingleObject`` (unchanged). + + If none of these mechanisms are available, the function falls back to the + traditional busy loop (non-blocking call and short sleeps). + (Contributed by Giampaolo Rodola in :gh:`83069`.) + + +symtable +-------- + +* Add :meth:`symtable.Function.get_cells` and :meth:`symtable.Symbol.is_cell` methods. + (Contributed by Yashp002 in :gh:`143504`.) + + +sys +--- + +* Add :data:`sys.abi_info` namespace to improve access to ABI information. + (Contributed by Klaus Zimmermann in :gh:`137476`.) + + +sys.monitoring +-------------- + +* The :ref:`other events ` + (:monitoring-event:`PY_THROW`, :monitoring-event:`PY_UNWIND`, + :monitoring-event:`RAISE`, :monitoring-event:`EXCEPTION_HANDLED`, and + :monitoring-event:`RERAISE`) can now be turned on and disabled on a per code + object basis. Returning :data:`~sys.monitoring.DISABLE` from a callback for + one of these events disables the event for the entire code object (for the + current tool), rather than raising :exc:`ValueError` as in prior versions. + (Contributed by Gabriele N. Tornetta in :gh:`146182`.) + + +tarfile +------- + +* :func:`~tarfile.data_filter` now normalizes symbolic link targets in order to + avoid path traversal attacks. + (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2025-4138`.) +* :func:`~tarfile.TarFile.extractall` now skips fixing up directory attributes + when a directory was removed or replaced by another kind of file. + (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2024-12718`.) +* :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` + now (re-)apply the extraction filter when substituting a link (hard or + symbolic) with a copy of another archive member, and when fixing up + directory attributes. + The former raises a new exception, :exc:`~tarfile.LinkFallbackError`. + (Contributed by Petr Viktorin for :cve:`2025-4330` and :cve:`2024-12718`.) +* :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` + no longer extract rejected members when + :func:`~tarfile.TarFile.errorlevel` is zero. + (Contributed by Matt Prodani and Petr Viktorin in :gh:`112887` + and :cve:`2025-4435`.) +* :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` + now replace slashes with backslashes in symlink targets on Windows to prevent + creation of corrupted links. + (Contributed by Christoph Walcher in :gh:`57911`.) + + +threading +--------- + +* Added :class:`~threading.serialize_iterator`, + :func:`~threading.synchronized_iterator`, + and :func:`~threading.concurrent_tee` to support concurrent access to + generators and iterators. + (Contributed by Raymond Hettinger in :gh:`124397`.) + + +timeit +------ + + .. _whatsnew315-color-timeit: + +* The output of the :mod:`timeit` command-line interface is colored by default. + This can be controlled with + :ref:`environment variables `. + (Contributed by Hugo van Kemenade in :gh:`146609`.) + +* The command-line interface now colorizes error tracebacks + by default. This can be controlled with + :ref:`environment variables `. + (Contributed by Yi Hong in :gh:`139374`.) + +* Make the target time of :meth:`timeit.Timer.autorange` configurable + and add ``--target-time`` option to the command-line interface. + (Contributed by Alessandro Cucci and Miikka Koskinen in :gh:`80642`.) + + +tkinter +------- + +* The :meth:`!tkinter.Text.search` method now supports two additional + arguments: *nolinestop* which allows the search to + continue across line boundaries; + and *strictlimits* which restricts the search to within the specified range. + (Contributed by Rihaan Meher in :gh:`130848`.) + +* A new method :meth:`!tkinter.Text.search_all` has been introduced. + This method allows for searching for all matches of a pattern + using Tcl's ``-all`` and ``-overlap`` options. + (Contributed by Rihaan Meher in :gh:`130848`.) + +* Added new methods :meth:`!pack_content`, :meth:`!place_content` and + :meth:`!grid_content` which use Tk commands with new names (introduced + in Tk 8.6) instead of :meth:`!*_slaves` methods which use Tk commands + with outdated names. + (Contributed by Serhiy Storchaka in :gh:`143754`.) + +* Added :class:`!Event` attributes :attr:`!user_data` for Tk virtual events + and :attr:`!detail` for ``Enter``, ``Leave``, ``FocusIn``, ``FocusOut``, + and ``ConfigureRequest`` events. + (Contributed by Matthias Kievernagel and Serhiy Storchaka in :gh:`47655`.) + + +tokenize +-------- + + .. _whatsnew315-color-tokenize: + +* The output of the :mod:`tokenize` :ref:`command-line interface + ` is colored by default. This can be controlled with + :ref:`environment variables `. + (Contributed by Hugo van Kemenade in :gh:`148991`.) + + +.. _whatsnew315-tomllib-1-1-0: + +tomllib +------- + +* The :mod:`tomllib` module now supports TOML 1.1.0. + This is a backwards compatible update, meaning that all valid TOML 1.0.0 + documents are parsed the same way. + + The changes, according to the `official TOML changelog`_, are: + + - Allow newlines and trailing commas in inline tables. + + Previously an inline table had to be on a single line and couldn't end + with a trailing comma. This is now relaxed so that the following is valid: + + .. code-block:: toml + + tbl = { + key = "a string", + moar-tbl = { + key = 1, + }, + } + + - Add ``\xHH`` notation to basic strings for codepoints under 255, + and the ``\e`` escape for the escape character: + + .. code-block:: toml + + null = "null byte: \x00; letter a: \x61" + csi = "\e[" + + - Seconds in datetime and time values are now optional. + The following are now valid: + + .. code-block:: toml + + dt = 2010-02-03 14:15 + t = 14:15 + + (Contributed by Taneli Hukkinen in :gh:`142956`.) + +.. _official TOML changelog: https://github.com/toml-lang/toml/blob/main/CHANGELOG.md + + +types +----- + +* Expose the write-through :func:`locals` proxy type + as :class:`types.FrameLocalsProxyType`. + This represents the type of the :attr:`frame.f_locals` attribute, + as described in :pep:`667`. + + +typing +------ + +.. _whatsnew315-typeform: + +* :pep:`747`: Add :data:`~typing.TypeForm`, a new special form for annotating + values that are themselves type expressions. + ``TypeForm[T]`` means "a type form object describing ``T`` (or a type + assignable to ``T``)". At runtime, ``TypeForm(x)`` simply returns ``x``, + which allows explicit annotation of type-form values without changing + behavior. + + This helps libraries that accept user-provided type expressions + (for example ``int``, ``str | None``, :class:`~typing.TypedDict` + classes, or ``list[int]``) expose precise signatures: + + .. code-block:: python + + from typing import Any, TypeForm + + def cast[T](typ: TypeForm[T], value: Any) -> T: ... + + (Contributed by Jelle Zijlstra in :gh:`145033`.) + +.. _whatsnew315-typeddict: + +* :pep:`728`: Add support in :class:`~typing.TypedDict` for the *closed* + and *extra_items* class arguments. A closed :class:`~typing.TypedDict` + does not allow extra keys beyond those specified in the class body, while + a :class:`~typing.TypedDict` with ``extra_items`` allows arbitrary extra + items where the values are of the specified type. (Contributed by Angela + Liss in :gh:`137840`.) + +* Code like ``class ExtraTypeVars(P1[S], Protocol[T, T2]): ...`` now raises + a :exc:`TypeError`, because ``S`` is not listed in ``Protocol`` parameters. + (Contributed by Nikita Sobolev in :gh:`137191`.) + +* Code like ``class B2(A[T2], Protocol[T1, T2]): ...`` now correctly handles + type parameters order: it is ``(T1, T2)``, not ``(T2, T1)`` + as it was incorrectly inferred at runtime before. + (Contributed by Nikita Sobolev in :gh:`137191`.) + +* :pep:`800`: Add :deco:`typing.disjoint_base`, a new decorator marking a class + as a disjoint base. This is an advanced feature primarily intended to allow + type checkers to faithfully reflect the runtime semantics of types defined + as builtins or in compiled extensions. If a class ``C`` is a disjoint base, then + child classes of that class cannot inherit from other disjoint bases that are + not parent or child classes of ``C``. (Contributed by Jelle Zijlstra in :gh:`148639`.) + +* :class:`~typing.TypeVarTuple` now accepts ``bound``, ``covariant``, + ``contravariant``, and ``infer_variance`` keyword arguments, matching the + interface of :class:`~typing.TypeVar` and :class:`~typing.ParamSpec`. + ``bound`` semantics remain undefined in the specification. + + +unicodedata +----------- + +* The Unicode database has been updated to Unicode 17.0.0. + +* Add :func:`unicodedata.isxidstart` and :func:`unicodedata.isxidcontinue` + functions to check whether a character can start or continue a + `Unicode Standard Annex #31 `_ identifier. + (Contributed by Stan Ulbrych in :gh:`129117`.) -* Add the ``'m'`` flag for :func:`dbm.gnu.open` which allows to disable - the use of :manpage:`mmap(2)`. - This may harm performance, but improve crash tolerance. - (Contributed by Serhiy Storchaka in :gh:`66234`.) +* Add the :func:`~unicodedata.iter_graphemes` + function to iterate over grapheme clusters according to rules defined in + `Unicode Standard Annex #29, "Unicode Text Segmentation" + `_. + Add :func:`~unicodedata.grapheme_cluster_break`, + :func:`~unicodedata.indic_conjunct_break` and + :func:`~unicodedata.extended_pictographic` functions to get the properties + of the character which are related to the above algorithm. + (Contributed by Serhiy Storchaka and Guillaume Sanchez in :gh:`74902`.) -difflib -------- +* Add :func:`~unicodedata.block` function to return the `Unicode block + `_ + assigned to a character. + (Contributed by Stan Ulbrych in :gh:`66802`.) -* Improved the styling of HTML diff pages generated by the :class:`difflib.HtmlDiff` - class, and migrated the output to the HTML5 standard. - (Contributed by Jiahao Li in :gh:`134580`.) +unittest +-------- -http.client ------------ +* :meth:`unittest.TestCase.assertLogs` will now accept a formatter + to control how messages are formatted. + (Contributed by Garry Cairns in :gh:`134567`.) -* A new *max_response_headers* keyword-only parameter has been added to - :class:`~http.client.HTTPConnection` and :class:`~http.client.HTTPSConnection` - constructors. This parameter overrides the default maximum number of allowed - response headers. - (Contributed by Alexander Enrique Urieles Nieto in :gh:`131724`.) +* :meth:`unittest.TestCase.assertWarns` and + :meth:`unittest.TestCase.assertWarnsRegex` no longer swallow warnings that + do not match the specified category or regex. + Nested context managers are now supported. + (Contributed by Serhiy Storchaka in :gh:`143231`.) -math ----- +urllib.parse +------------ -* Add :func:`math.isnormal` and :func:`math.issubnormal` functions. - (Contributed by Sergey B Kirpichev in :gh:`132908`.) +* Add the *missing_as_none* parameter to :func:`~urllib.parse.urlsplit`, + :func:`~urllib.parse.urlparse` and :func:`~urllib.parse.urldefrag` functions. + Add the *keep_empty* parameter to :func:`~urllib.parse.urlunsplit` and + :func:`~urllib.parse.urlunparse` functions. + This allows distinguishing between empty and undefined URI components + and preserving empty components. + (Contributed by Serhiy Storchaka in :gh:`67041`.) -* Add :func:`math.fmax`, :func:`math.fmin` and :func:`math.signbit` functions. - (Contributed by Bénédikt Tran in :gh:`135853`.) +venv +---- -os.path -------- +* On POSIX platforms, platlib directories will be created if needed when + creating virtual environments, instead of using a ``lib64 -> lib`` symlink. + This means purelib and platlib of virtual environments no longer share the + same ``lib`` directory on platforms where :data:`sys.platlibdir` is not + equal to ``lib``. + (Contributed by Rui Xi in :gh:`133951`.) -* The *strict* parameter to :func:`os.path.realpath` accepts a new value, - :data:`os.path.ALLOW_MISSING`. - If used, errors other than :exc:`FileNotFoundError` will be re-raised; - the resulting path can be missing but it will be free of symlinks. - (Contributed by Petr Viktorin for :cve:`2025-4517`.) +warnings +-------- -shelve ------- +* Improve filtering by module in :func:`warnings.warn_explicit` if no *module* + argument is passed. + It now tests the module regular expression in the warnings filter not only + against the filename with ``.py`` stripped, but also against module names + constructed starting from different parent directories of the filename + (with ``/__init__.py``, ``.py`` and, on Windows, ``.pyw`` stripped). + (Contributed by Serhiy Storchaka in :gh:`135801`.) -* Added new :meth:`!reorganize` method to :mod:`shelve` used to recover unused free - space previously occupied by deleted entries. - (Contributed by Andrea Oliveri in :gh:`134004`.) +wave +---- -sqlite3 -------- +* Added support for IEEE floating-point WAVE audio + (``WAVE_FORMAT_IEEE_FLOAT``) in :mod:`wave`. -* The :ref:`command-line interface ` has several new features: +* Added :meth:`wave.Wave_read.getformat`, :meth:`wave.Wave_write.getformat`, + and :meth:`wave.Wave_write.setformat` for explicit frame format handling. - * SQL keyword completion on . - (Contributed by Long Tan in :gh:`133393`.) +* :meth:`wave.Wave_write.setparams` accepts both 7-item tuples including + ``format`` and 6-item tuples for backwards compatibility (defaulting to + ``WAVE_FORMAT_PCM``). - * Prompts, error messages, and help text are now colored. - This is enabled by default, see :ref:`using-on-controlling-color` for - details. - (Contributed by Stan Ulbrych and Łukasz Langa in :gh:`133461`) +* ``WAVE_FORMAT_IEEE_FLOAT`` output now includes a ``fact`` chunk, + as required for non-PCM WAVE formats. +(Contributed by Lionel Koenig and Michiel W. Beijen in :gh:`60729`.) -ssl ---- -* Indicate through :data:`ssl.HAS_PSK_TLS13` whether the :mod:`ssl` module - supports "External PSKs" in TLSv1.3, as described in RFC 9258. - (Contributed by Will Childs-Klein in :gh:`133624`.) +webbrowser +---------- +* On macOS, the new :class:`!webbrowser.MacOS` class opens URLs via + :program:`/usr/bin/open` instead of constructing and executing AppleScript + via :program:`osascript`. The default browser is detected from the + LaunchServices preferences file using :mod:`plistlib`, with + :class:`!com.apple.Safari` as the fallback on fresh installations. + For non-HTTP(S) URLs, :program:`open -b ` is used to route the + URL through a browser rather than the OS file handler, preventing + file injection attacks. + (Contributed by Jeff Lyon in :gh:`137586`.) -tarfile -------- -* :func:`~tarfile.data_filter` now normalizes symbolic link targets in order to - avoid path traversal attacks. - (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2025-4138`.) -* :func:`~tarfile.TarFile.extractall` now skips fixing up directory attributes - when a directory was removed or replaced by another kind of file. - (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2024-12718`.) -* :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` - now (re-)apply the extraction filter when substituting a link (hard or - symbolic) with a copy of another archive member, and when fixing up - directory attributes. - The former raises a new exception, :exc:`~tarfile.LinkFallbackError`. - (Contributed by Petr Viktorin for :cve:`2025-4330` and :cve:`2024-12718`.) -* :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` - no longer extract rejected members when - :func:`~tarfile.TarFile.errorlevel` is zero. - (Contributed by Matt Prodani and Petr Viktorin in :gh:`112887` - and :cve:`2025-4435`.) +xml +--- +* Add the :func:`xml.is_valid_name` function to check + whether a string can be used as an element or attribute name in XML. + (Contributed by Serhiy Storchaka in :gh:`139489`.) -types ------- +* Add the :func:`xml.is_valid_text` function, which allows to check + whether a string can be used in the XML document. + (Contributed by Serhiy Storchaka in :gh:`139489`.) -* Expose the write-through :func:`locals` proxy type - as :data:`types.FrameLocalsProxyType`. - This represents the type of the :attr:`frame.f_locals` attribute, - as described in :pep:`667`. +xml.parsers.expat +----------------- + +* Add :meth:`~xml.parsers.expat.xmlparser.SetAllocTrackerActivationThreshold` + and :meth:`~xml.parsers.expat.xmlparser.SetAllocTrackerMaximumAmplification` + to :ref:`xmlparser ` objects to tune protections against + disproportional amounts of dynamic memory usage from within an Expat parser. + (Contributed by Bénédikt Tran in :gh:`90949`.) -unittest --------- +* Add :meth:`~xml.parsers.expat.xmlparser.SetBillionLaughsAttackProtectionActivationThreshold` + and :meth:`~xml.parsers.expat.xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification` + to :ref:`xmlparser ` objects to tune protections against + `billion laughs`_ attacks. + (Contributed by Bénédikt Tran in :gh:`90949`.) -* :func:`unittest.TestCase.assertLogs` will now accept a formatter - to control how messages are formatted. - (Contributed by Garry Cairns in :gh:`134567`.) + .. _billion laughs: https://en.wikipedia.org/wiki/Billion_laughs_attack zlib @@ -344,36 +1873,164 @@ zlib Optimizations ============= -module_name ------------ - -* TODO +* ``mimalloc`` is now used as the default allocator for + raw memory allocations such as via :c:func:`PyMem_RawMalloc` + for better performance on :term:`free-threaded builds `. + (Contributed by Kumar Aditya in :gh:`144914`.) +base64 & binascii +----------------- -Deprecated -========== +* CPython's underlying base64 implementation now encodes 2x faster and decodes 3x + faster thanks to simple CPU pipelining optimizations. + (Contributed by Gregory P. Smith and Serhiy Storchaka in :gh:`143262`.) -hashlib -------- +* Implementation for Ascii85, Base85, and Z85 encoding has been rewritten in C. + Encoding and decoding is now two orders of magnitude faster and consumes + two orders of magnitude less memory. + (Contributed by James Seo and Serhiy Storchaka in :gh:`101178`.) -* In hash function constructors such as :func:`~hashlib.new` or the - direct hash-named constructors such as :func:`~hashlib.md5` and - :func:`~hashlib.sha256`, their optional initial data parameter could - also be passed a keyword argument named ``data=`` or ``string=`` in - various :mod:`hashlib` implementations. +* Implementation for Base32 has been rewritten in C. + Encoding and decoding is now two orders of magnitude faster. + (Contributed by James Seo in :gh:`146192`.) - Support for the ``string`` keyword argument name is now deprecated and - is slated for removal in Python 3.19. Prefer passing the initial data as - a positional argument for maximum backwards compatibility. - (Contributed by Bénédikt Tran in :gh:`134978`.) +csv +--- +* :meth:`csv.Sniffer.sniff` delimiter detection is now up to 1.6x faster. + (Contributed by Maurycy Pawłowski-Wieroński in :gh:`137628`.) + + +.. _whatsnew315-jit: + +Upgraded JIT compiler +--------------------- + +Results from the `pyperformance `__ +benchmark suite report +`8-9% `__ +geometric mean performance improvement for the JIT over the standard CPython +interpreter built with all optimizations enabled on x86-64 Linux. On AArch64 +macOS, the JIT has a +`12-13% `__ +speedup over the :ref:`tail calling interpreter ` +with all optimizations enabled. The speedups for JIT +builds versus no JIT builds range from roughly 15% slowdown to over +100% speedup (ignoring the ``unpack_sequence`` microbenchmark) on +x86-64 Linux and AArch64 macOS systems. + +.. attention:: + These results are not yet final. + +The major upgrades to the JIT are: + +* LLVM 21 build-time dependency +* New tracing frontend +* Basic register allocation in the JIT +* More JIT optimizations +* GDB and GNU ``backtrace()`` unwinding support +* Better machine code generation + +.. rubric:: LLVM 21 build-time dependency + +The JIT compiler now uses LLVM 21 for build-time stencil generation. As +always, LLVM is only needed when building CPython with the JIT enabled; +end users running Python do not need LLVM installed. Instructions for +installing LLVM can be found in the `JIT compiler documentation +`__ +for all supported platforms. +(Contributed by Savannah Ostrowski in :gh:`140973`.) + +.. rubric:: A new tracing frontend + +The JIT compiler now supports significantly more bytecode operations and +control flow than in Python 3.14, enabling speedups on a wider variety of +code. For example, simple Python object creation is now understood by the +3.15 JIT compiler. Overloaded operations and generators are also partially +supported. This was made possible by an overhauled JIT tracing frontend +that records actual execution paths through code, rather than estimating +them as the previous implementation did. +(Contributed by Ken Jin in :gh:`139109`. Support for Windows added by +Mark Shannon in :gh:`141703`.) + +.. rubric:: Basic register allocation in the JIT + +A basic form of register allocation has been added to the JIT compiler's +optimizer. This allows the JIT compiler to avoid certain stack operations +altogether and instead operate on registers. This allows the JIT to produce +more efficient traces by avoiding reads and writes to memory. +(Contributed by Mark Shannon in :gh:`135379`.) + +.. rubric:: More JIT optimizations + +More `constant-propagation `__ +is now performed. This means when the JIT compiler detects that certain user +code results in constants, the code can be simplified by the JIT. +(Contributed by Ken Jin and Savannah Ostrowski in :gh:`132732`.) + +:term:`Reference count`\ s are avoided whenever it is safe to do so. This generally +reduces the cost of most operations in Python. +(Contributed by Ken Jin, Donghee Na, Zheao Li, Hai Zhu, Savannah Ostrowski, +Reiden Ong, Noam Cohen, Tomas Roun, PuQing, Cajetan Rodrigues, and Sacul in :gh:`134584`.) + +By tracking unique references to objects, the JIT optimizer can now eliminate +reference count updates and perform in-place operations on ints and floats. +(Contributed by Reiden Ong, and Pieter Eendebak in :gh:`143414` and :gh:`146306`.) + +The JIT optimizer now supports significantly more operations than in 3.14. +(Contributed by Kumar Aditya, Ken Jin, Jiahao Li, and Sacul in :gh:`131798`.) + +.. rubric:: GDB and GNU ``backtrace()`` unwinding support + +The JIT compiler now publishes unwind information for generated machine code to the +GDB interface on supported Linux ELF platforms. When libgcc frame +registration is available, the same unwind information is also registered for +GNU ``backtrace()`` stack walkers. This allows native debuggers, crash +handlers, and diagnostic tools using these mechanisms to unwind through JIT +frames instead of stopping at generated code. +(Contributed by Diego Russo and Pablo Galindo Salgado in :gh:`146071` and +:gh:`149104`.) + +.. rubric:: Better machine code generation + +The JIT compiler's machine code generator now produces better machine code +for x86-64 and AArch64 macOS and Linux targets. In general, users should +experience lower memory usage for generated machine code and more efficient +machine code versus 3.14. +(Contributed by Brandt Bucher in :gh:`136528` and :gh:`135905`. +Implementation for AArch64 contributed by Mark Shannon in :gh:`139855`. +Additional optimizations for AArch64 contributed by Mark Shannon and +Diego Russo in :gh:`140683` and :gh:`142305`.) + +.. rubric:: Maintainability + +The JIT optimizer's operations have been simplified. +This was made possible by a refactoring of JIT data structures. +(Contributed by Zhongtian Zheng in :gh:`148211` and Hai Zhu in :gh:`143421`.) -.. Add deprecations above alphabetically, not here at the end. Removed -======= +======== + +ast +--- + +* The constructors of :ref:`AST nodes ` now raise a :exc:`TypeError` + when a required argument is omitted or when a keyword argument that does not + map to a field on the AST node is passed. These cases had previously raised a + :exc:`DeprecationWarning` since Python 3.13. + (Contributed by Brian Schubert and Jelle Zijlstra in :gh:`137600` and :gh:`105858`.) + + +collections.abc +--------------- + +* :class:`collections.abc.ByteString` has been removed from + ``collections.abc.__all__``. :class:`!collections.abc.ByteString` has been + deprecated since Python 3.12, and is scheduled for removal in Python 3.17. + ctypes ------ @@ -382,6 +2039,30 @@ ctypes which has been deprecated since Python 3.13. (Contributed by Bénédikt Tran in :gh:`133866`.) +* Change the :py:attr:`~ctypes._SimpleCData._type_` of + :class:`~ctypes.c_float_complex`, :class:`~ctypes.c_double_complex` and + :class:`~ctypes.c_longdouble_complex` from ``F``, ``D`` and ``G`` to ``Zf``, + ``Zd`` and ``Zg`` for compatibility with numpy. + (Contributed by Victor Stinner in :gh:`148675`.) + + +datetime +-------- + +* :meth:`~datetime.datetime.strptime` now raises :exc:`ValueError` when the + format string contains ``%d`` (day of month) without a year directive. + This has been deprecated since Python 3.13. + (Contributed by Stan Ulbrych and Gregory P. Smith in :gh:`70647`.) + + +glob +---- + +* Removed the undocumented :func:`!glob.glob0` and :func:`!glob.glob1` + functions, which have been deprecated since Python 3.13. Use + :func:`glob.glob` and pass a directory to its *root_dir* argument instead. + (Contributed by Barney Gale in :gh:`137466`.) + http.server ----------- @@ -392,9 +2073,21 @@ http.server (Contributed by Bénédikt Tran in :gh:`133810`.) +importlib.resources +------------------- + +* Removed deprecated ``package`` parameter + from :func:`importlib.resources.files` function. + (Contributed by Semyon Moroz in :gh:`138044`.) + + pathlib ------- +* :meth:`pathlib.Path.mkdir` now has a *parent_mode* parameter that allows + specifying the mode for intermediate directories when ``parents=True``. + (Contributed by Gregory P. Smith in :gh:`86533`.) + * Removed deprecated :meth:`!pathlib.PurePath.is_reserved`. Use :func:`os.path.isreserved` to detect reserved paths on Windows. (Contributed by Nikita Sobolev in :gh:`133875`.) @@ -431,9 +2124,21 @@ threading (Contributed by Bénédikt Tran in :gh:`134087`.) +types +----- + +* Removed deprecated in :pep:`626` since Python 3.12 + :attr:`!codeobject.co_lnotab` from :class:`types.CodeType`. + (Contributed by Nikita Sobolev in :gh:`134690`.) + + typing ------ +* :class:`typing.ByteString` has been removed from ``typing.__all__``. + :class:`!typing.ByteString` has been deprecated since Python 3.9, and is + scheduled for removal in Python 3.17. + * The undocumented keyword argument syntax for creating :class:`~typing.NamedTuple` classes (for example, ``Point = NamedTuple("Point", x=int, y=int)``) is no longer supported. @@ -441,11 +2146,14 @@ typing (Contributed by Bénédikt Tran in :gh:`133817`.) * Using ``TD = TypedDict("TD")`` or ``TD = TypedDict("TD", None)`` to - construct a :class:`~typing.TypedDict` type with zero field is no + construct a :class:`~typing.TypedDict` type with zero fields is no longer supported. Use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})`` instead. (Contributed by Bénédikt Tran in :gh:`133823`.) +* Deprecated :deco:`!typing.no_type_check_decorator` has been removed. + (Contributed by Nikita Sobolev in :gh:`133601`.) + wave ---- @@ -456,20 +2164,192 @@ wave (Contributed by Bénédikt Tran in :gh:`133873`.) -Porting to Python 3.15 -====================== +zipimport +--------- -This section lists previously described changes and other bugfixes -that may require changes to your code. +* Remove deprecated :meth:`!zipimport.zipimporter.load_module`. + Use :meth:`zipimport.zipimporter.exec_module` instead. + (Contributed by Jiahao Li in :gh:`133656`.) -Build changes -============= +Deprecated +========== -* Removed implicit fallback to the bundled copy of the ``libmpdec`` library. - Now this should be explicitly enabled with :option:`--with-system-libmpdec` - set to ``no`` or with :option:`!--without-system-libmpdec`. - (Contributed by Sergey B Kirpichev in :gh:`115119`.) +New deprecations +---------------- + +* :mod:`ast` + + * Creating instances of abstract AST nodes (such as :class:`ast.AST` + or :class:`!ast.expr`) is deprecated and will raise an error in Python 3.20. + + (Contributed by Brian Schubert in :gh:`116021`.) + +* :mod:`base64`: + + * Accepting the ``+`` and ``/`` characters with an alternative alphabet in + :func:`~base64.b64decode` and :func:`~base64.urlsafe_b64decode` is now + deprecated. + In future Python versions they will be errors in the strict mode and + discarded in the non-strict mode. + (Contributed by Serhiy Storchaka in :gh:`125346`.) + +* CLI: + + * Deprecate :option:`-b` and :option:`!-bb` command-line options + and schedule them to become no-ops in Python 3.17. + These were primarily helpers for the Python 2 -> 3 transition. + Starting with Python 3.17, no :exc:`BytesWarning` will be raised + for these cases; use a type checker instead. + + (Contributed by Nikita Sobolev in :gh:`136355`.) + +* :mod:`collections.abc` + + * The following statements now cause ``DeprecationWarning``\ s to be emitted + at runtime: + + * ``from collections.abc import ByteString`` + * ``import collections.abc; collections.abc.ByteString``. + + ``DeprecationWarning``\ s were already emitted if + :class:`collections.abc.ByteString` was subclassed or used as the second + argument to :func:`isinstance` or :func:`issubclass`, but warnings were not + previously emitted if it was merely imported or accessed from the + :mod:`!collections.abc` module. + + +* :mod:`hashlib`: + + * In hash function constructors such as :func:`~hashlib.new` or the + direct hash-named constructors such as :func:`~hashlib.md5` and + :func:`~hashlib.sha256`, the optional initial data parameter could + also be passed as a keyword argument named ``data=`` or ``string=`` in + various :mod:`hashlib` implementations. + + Support for the ``string`` keyword argument name is now deprecated and + is slated for removal in Python 3.19. Prefer passing the initial data as + a positional argument for maximum backwards compatibility. + + (Contributed by Bénédikt Tran in :gh:`134978`.) + + +* :mod:`http.cookies`: + + * :meth:`Morsel.js_output ` and + :meth:`BaseCookie.js_output ` are + deprecated and will be removed in Python 3.19. Use + :meth:`Morsel.output ` or + :meth:`BaseCookie.output ` instead. + (Contributed by kishorhange111 in :gh:`148849`.) + + +* :mod:`imaplib`: + + * Altering :attr:`IMAP4.file ` is now deprecated + and slated for removal in Python 3.19. This property is now unused + and changing its value does *not* explicitly close the current file. + + +* :mod:`re`: + + * :func:`re.match` and :meth:`re.Pattern.match` are now + :term:`soft deprecated` in favor of the new :func:`re.prefixmatch` and + :meth:`re.Pattern.prefixmatch` APIs, which have been added as alternate, + more explicit names. These are intended to be used to alleviate confusion + around what *match* means by following the Zen of Python's *"Explicit is + better than implicit"* mantra. Most other language regular expression + libraries use an API named *match* to mean what Python has always called + *search*. + + We **do not** plan to remove the older :func:`!match` name, as it has been + used in code for over 30 years. Code supporting older versions of Python + should continue to use :func:`!match`, while new code should prefer + :func:`!prefixmatch`. See :ref:`prefixmatch-vs-match`. + + (Contributed by Gregory P. Smith in :gh:`86519` and + Hugo van Kemenade in :gh:`148100`.) + + +* :mod:`struct`: + + * Calling ``Struct.__new__()`` without a required argument is now + deprecated and will be removed in Python 3.20. Calling the + :meth:`~object.__init__` method on an initialized :class:`~struct.Struct` + object is deprecated and will be removed in Python 3.20. + + (Contributed by Sergey B Kirpichev and Serhiy Storchaka in :gh:`143715`.) + +* :mod:`typing`: + + * The following statements now cause ``DeprecationWarning``\ s to be emitted + at runtime: + + * ``from typing import ByteString`` + * ``import typing; typing.ByteString``. + + ``DeprecationWarning``\ s were already emitted if :class:`typing.ByteString` + was subclassed or used as the second argument to :func:`isinstance` or + :func:`issubclass`, but warnings were not previously emitted if it was + merely imported or accessed from the :mod:`!typing` module. + + +* :mod:`webbrowser`: + + * :class:`!webbrowser.MacOSXOSAScript` is deprecated in favour of + :class:`!webbrowser.MacOS` and scheduled for removal in Python 3.17. + (Contributed by Jeff Lyon in :gh:`137586`.) + +* ``__version__`` + + * The ``__version__``, ``version`` and ``VERSION`` attributes have been + deprecated in these standard library modules and will be removed in + Python 3.20. Use :py:data:`sys.version_info` instead. + + - :mod:`argparse` + - :mod:`csv` + - :mod:`ctypes` + - :mod:`!ctypes.macholib` + - :mod:`decimal` (use :data:`decimal.SPEC_VERSION` instead) + - :mod:`http.server` + - :mod:`imaplib` + - :mod:`ipaddress` + - :mod:`json` + - :mod:`logging` (``__date__`` also deprecated) + - :mod:`optparse` + - :mod:`pickle` + - :mod:`platform` + - :mod:`re` + - :mod:`socketserver` + - :mod:`tabnanny` + - :mod:`tarfile` + - :mod:`tkinter.font` + - :mod:`tkinter.ttk` + - :mod:`wsgiref.simple_server` + - :mod:`xml.etree.ElementTree` + - :mod:`!xml.sax.expatreader` + - :mod:`xml.sax.handler` + - :mod:`zlib` + + (Contributed by Hugo van Kemenade and Stan Ulbrych in :gh:`76007`.) + +.. Add deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/pending-removal-in-3.16.rst + +.. include:: ../deprecations/pending-removal-in-3.17.rst + +.. include:: ../deprecations/pending-removal-in-3.18.rst + +.. include:: ../deprecations/pending-removal-in-3.19.rst + +.. include:: ../deprecations/pending-removal-in-3.20.rst + +.. include:: ../deprecations/pending-removal-in-3.21.rst + +.. include:: ../deprecations/pending-removal-in-future.rst + +.. include:: ../deprecations/soft-deprecations.rst C API changes @@ -478,6 +2358,25 @@ C API changes New features ------------ +* Add :c:func:`PyArg_ParseArray` and :c:func:`PyArg_ParseArrayAndKeywords` + functions to parse arguments of functions using the :c:macro:`METH_FASTCALL` + calling convention. + (Contributed by Victor Stinner in :gh:`144175`.) + +* Add the following functions for the new :class:`frozendict` type: + + * :c:func:`PyAnyDict_Check` + * :c:func:`PyAnyDict_CheckExact` + * :c:func:`PyFrozenDict_Check` + * :c:func:`PyFrozenDict_CheckExact` + * :c:func:`PyFrozenDict_New` + + (Contributed by Victor Stinner in :gh:`141510`.) + +* Add :c:func:`PyObject_CallFinalizerFromDealloc` function to the limited C + API. + (Contributed by Victor Stinner in :gh:`146063`.) + * Add :c:func:`PySys_GetAttr`, :c:func:`PySys_GetAttrString`, :c:func:`PySys_GetOptionalAttr`, and :c:func:`PySys_GetOptionalAttrString` functions as replacements for :c:func:`PySys_GetObject`. @@ -485,39 +2384,135 @@ New features * Add :c:type:`PyUnstable_Unicode_GET_CACHED_HASH` to get the cached hash of a string. See the documentation for caveats. - (Contributed by Petr Viktorin in :gh:`131510`) - - -Porting to Python 3.15 ----------------------- - -* :class:`sqlite3.Connection` APIs has been cleaned up. - - * All parameters of :func:`sqlite3.connect` except *database* are now keyword-only. - * The first three parameters of methods :meth:`~sqlite3.Connection.create_function` - and :meth:`~sqlite3.Connection.create_aggregate` are now positional-only. - * The first parameter of methods :meth:`~sqlite3.Connection.set_authorizer`, - :meth:`~sqlite3.Connection.set_progress_handler` and - :meth:`~sqlite3.Connection.set_trace_callback` is now positional-only. - - (Contributed by Serhiy Storchaka in :gh:`133595`.) - -* Private functions promoted to public C APIs: - - The |pythoncapi_compat_project| can be used to get most of these new - functions on Python 3.14 and older. - + (Contributed by Petr Viktorin in :gh:`131510`.) + +* Add API for checking an extension module's ABI compatibility: + :c:data:`Py_mod_abi`, :c:func:`PyABIInfo_Check`, + and :c:macro:`PyABIInfo_VAR`. + (Contributed by Petr Viktorin in :gh:`137210`.) + +.. _whatsnew315-pybyteswriter: + +* Implement :pep:`782`, the :ref:`PyBytesWriter API `. + Add functions: + + * :c:func:`PyBytesWriter_Create` + * :c:func:`PyBytesWriter_Discard` + * :c:func:`PyBytesWriter_FinishWithPointer` + * :c:func:`PyBytesWriter_FinishWithSize` + * :c:func:`PyBytesWriter_Finish` + * :c:func:`PyBytesWriter_Format` + * :c:func:`PyBytesWriter_GetData` + * :c:func:`PyBytesWriter_GetSize` + * :c:func:`PyBytesWriter_GrowAndUpdatePointer` + * :c:func:`PyBytesWriter_Grow` + * :c:func:`PyBytesWriter_Resize` + * :c:func:`PyBytesWriter_WriteBytes` + + (Contributed by Victor Stinner in :gh:`129813`.) + +* :c:type:`PyCriticalSection` and related functions are added to the Stable + ABI. + + (Contributed in :gh:`149227`.) + +* Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating + a module from a *spec* and *initfunc*. + (Contributed by Itamar Oren in :gh:`116146`.) + +* Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. + (Contributed by Victor Stinner in :gh:`111489`.) + +* Add a new module export hook, + :c:func:`PyModExport_* `. + + (Contributed by Petr Viktorin in :pep:`793` and :gh:`140550`.) + +* Add functions that are guaranteed to be safe for use in + :c:member:`~PyTypeObject.tp_traverse` handlers: + :c:func:`PyObject_GetTypeData_DuringGC`, + :c:func:`PyObject_GetItemData_DuringGC`, + :c:func:`PyType_GetModuleState_DuringGC`, + :c:func:`PyModule_GetState_DuringGC`, :c:func:`PyModule_GetToken_DuringGC`, + :c:func:`PyType_GetBaseByToken_DuringGC`, + :c:func:`PyType_GetModule_DuringGC`, + :c:func:`PyType_GetModuleByToken_DuringGC`. + (Contributed by Petr Viktorin in :gh:`145925`.) + +* Add :c:func:`PyObject_Dump` to dump an object to ``stderr``. + It should only be used for debugging. + (Contributed by Victor Stinner in :gh:`141070`.) + +* Implement :pep:`820`: ``PySlot`` -- Unified slot system for the C API. + See :ref:`capi-slots` for documentation. + + This adds: + + * The :c:type:`PySlot` struct; + * the :c:func:`PyType_FromSlots` function; + * new slot IDs: :c:macro:`Py_slot_end`, :c:macro:`Py_slot_invalid`; + :c:macro:`Py_slot_subslots`, :c:macro:`Py_tp_slots`, + :c:macro:`Py_mod_slots`; + :c:macro:`Py_tp_name`, :c:macro:`Py_tp_basicsize`, + :c:macro:`Py_tp_extra_basicsize`, :c:macro:`Py_tp_itemsize`, + :c:macro:`Py_tp_flags`, :c:macro:`Py_tp_metaclass`, + :c:macro:`Py_tp_module`; + * convenience macros: :c:macro:`PySlot_DATA`, :c:macro:`PySlot_FUNC`, + :c:macro:`PySlot_SIZE`, :c:macro:`PySlot_INT64`, :c:macro:`PySlot_UINT64`, + :c:macro:`PySlot_STATIC_DATA`, :c:macro:`PySlot_END`, + :c:macro:`PySlot_PTR`, :c:macro:`PySlot_PTR_STATIC`. + + The :c:func:`PyModule_FromSlotsAndSpec` function and + ``PyModExport`` :ref:`module export hook ` also + use the new :c:type:`!PySlot` struct. + + These following functions are :term:`soft deprecated`: + + * :c:func:`PyType_FromSpec` + * :c:func:`PyType_FromSpecWithBases` + * :c:func:`PyType_FromModuleAndSpec` + * :c:func:`PyType_FromMetaclass` + * :c:func:`PyModule_FromDefAndSpec` + * :c:func:`PyModule_FromDefAndSpec2` + * :c:func:`PyModule_ExecDef` + + + The slots :c:macro:`Py_tp_bases` and :c:macro:`Py_tp_base` are now + equivalent: they can be set either to a single type or a tuple of types. + The :c:macro:`Py_tp_bases` slot is preferred; the other is ignored if both + are specified. + + (Contributed by Petr Viktorin in :gh:`149044`.) + +* Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and + :c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set + the stack protection base address and stack protection size of a Python + thread state. + (Contributed by Victor Stinner in :gh:`139653`.) + +* Add :c:func:`PyUnstable_SetImmortal` C-API function to mark objects as :term:`immortal`. + (Contributed by Kumar Aditya in :gh:`143300`.) + +* Restore private provisional ``_Py_InitializeMain()`` function removed in + Python 3.14. + (Contributed by Victor Stinner in :gh:`142417`.) + +* Add :c:func:`PyUnstable_DumpTraceback` and + :c:func:`PyUnstable_DumpTracebackThreads` functions to output Python + stacktraces. + (Contributed by Alex Malyshev in :gh:`145559`.) + +Changed C APIs +-------------- -Deprecated C APIs ------------------ +* If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` or :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` + flag is set then :c:macro:`Py_TPFLAGS_HAVE_GC` must be set too. + (Contributed by Sergey Miryanov in :gh:`134786`.) -* For unsigned integer formats in :c:func:`PyArg_ParseTuple`, - accepting Python integers with value that is larger than the maximal value - for the C type or less than the minimal value for the corresponding - signed integer type of the same size is now deprecated. - (Contributed by Serhiy Storchaka in :gh:`132629`.) +* :c:macro:`PyDateTime_IMPORT` is now thread safe. Code that directly checks ``PyDateTimeAPI`` + for ``NULL`` should be updated to call :c:macro:`PyDateTime_IMPORT` instead. + (Contributed by Kumar Aditya in :gh:`141563`.) -.. Add C API deprecations above alphabetically, not here at the end. Removed C APIs -------------- @@ -527,20 +2522,30 @@ Removed C APIs * :c:func:`!PyUnicode_AsDecodedObject`: Use :c:func:`PyCodec_Decode` instead. * :c:func:`!PyUnicode_AsDecodedUnicode`: - Use :c:func:`PyCodec_Decode` instead; Note that some codecs (for example, "base64") + Use :c:func:`PyCodec_Decode` instead; note that some codecs (for example, "base64") may return a type other than :class:`str`, such as :class:`bytes`. * :c:func:`!PyUnicode_AsEncodedObject`: Use :c:func:`PyCodec_Encode` instead. * :c:func:`!PyUnicode_AsEncodedUnicode`: - Use :c:func:`PyCodec_Encode` instead; Note that some codecs (for example, "base64") + Use :c:func:`PyCodec_Encode` instead; note that some codecs (for example, "base64") may return a type other than :class:`bytes`, such as :class:`str`. - (Contributed by Stan Ulbrych in :gh:`133612`) + (Contributed by Stan Ulbrych in :gh:`133612`.) * :c:func:`!PyImport_ImportModuleNoBlock`: deprecated alias of :c:func:`PyImport_ImportModule`. (Contributed by Bénédikt Tran in :gh:`133644`.) +* :c:func:`!PyWeakref_GetObject` and :c:macro:`!PyWeakref_GET_OBJECT`: + use :c:func:`PyWeakref_GetRef` instead. The |pythoncapi_compat_project| + can be used to get :c:func:`!PyWeakref_GetRef` on Python 3.12 and older. + (Contributed by Bénédikt Tran in :gh:`133644`.) + +* Remove deprecated :c:func:`!PySys_ResetWarnOptions`. + Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. + + (Contributed by Nikita Sobolev in :gh:`138886`.) + The following functions are removed in favor of :c:func:`PyConfig_Get`. The |pythoncapi_compat_project| can be used to get :c:func:`!PyConfig_Get` on Python 3.13 and older. @@ -577,3 +2582,191 @@ on Python 3.13 and older. .. |pythoncapi_compat_project| replace:: |pythoncapi_compat_project_link|_ .. |pythoncapi_compat_project_link| replace:: pythoncapi-compat project .. _pythoncapi_compat_project_link: https://github.com/python/pythoncapi-compat + + +Deprecated C APIs +----------------- + +* Deprecate :pep:`456` support for providing an external definition + of the string hashing scheme. Removal is scheduled for Python 3.19. + + Previously, embedders could define :c:macro:`Py_HASH_ALGORITHM` to be + ``Py_HASH_EXTERNAL`` to indicate that the hashing scheme was provided + externally but this feature was undocumented, untested and most likely + unused. + + (Contributed by Bénédikt Tran in :gh:`141226`.) + +* For unsigned integer formats in :c:func:`PyArg_ParseTuple`, + accepting Python integers with value that is larger than the maximal value + for the C type or less than the minimal value for the corresponding + signed integer type of the same size is now deprecated. + (Contributed by Serhiy Storchaka in :gh:`132629`.) + +* :c:func:`PyBytes_FromStringAndSize(NULL, len) ` + and :c:func:`_PyBytes_Resize` are :term:`soft deprecated`, + use the :c:type:`PyBytesWriter` API instead. + (Contributed by Victor Stinner in :gh:`129813`.) + +* :c:func:`!_PyObject_CallMethodId`, :c:func:`!_PyObject_GetAttrId` and + :c:func:`!_PyUnicode_FromId` are deprecated since 3.15 and will be removed in + 3.20. Instead, use :c:func:`PyUnicode_InternFromString()` and cache the result in + the module state, then call :c:func:`PyObject_CallMethod` or + :c:func:`PyObject_GetAttr`. + (Contributed by Victor Stinner in :gh:`141049`.) + +* Deprecate :c:member:`~PyComplexObject.cval` field of the + :c:type:`PyComplexObject` type. + Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex` + to convert a Python complex number to/from the C :c:type:`Py_complex` + representation. + (Contributed by Sergey B Kirpichev in :gh:`128813`.) + +* Functions :c:func:`_Py_c_sum`, :c:func:`_Py_c_diff`, :c:func:`_Py_c_neg`, + :c:func:`_Py_c_prod`, :c:func:`_Py_c_quot`, :c:func:`_Py_c_pow` and + :c:func:`_Py_c_abs` are :term:`soft deprecated`. + (Contributed by Sergey B Kirpichev in :gh:`128813`.) + +* :c:member:`~PyConfig.bytes_warning` is deprecated + since 3.15 and will be removed in 3.17. + (Contributed by Nikita Sobolev in :gh:`136355`.) + +* :c:macro:`!Py_INFINITY` macro is :term:`soft deprecated`, + use the C11 standard ```` :c:macro:`!INFINITY` instead. + (Contributed by Sergey B Kirpichev in :gh:`141004`.) + +* The following macros are :term:`soft deprecated`: + + - :c:macro:`Py_ALIGNED`: Prefer ``alignas`` instead. + - :c:macro:`PY_FORMAT_SIZE_T`: Use ``"z"`` directly. + - :c:macro:`Py_LL` and :c:macro:`Py_ULL`: + Use standard suffixes, ``LL`` and ``ULL``. + - :c:macro:`PY_LONG_LONG`, :c:macro:`PY_LLONG_MIN`, :c:macro:`PY_LLONG_MAX`, + :c:macro:`PY_ULLONG_MAX`, :c:macro:`PY_INT32_T`, :c:macro:`PY_UINT32_T`, + :c:macro:`PY_INT64_T`, :c:macro:`PY_UINT64_T`, :c:macro:`PY_SIZE_MAX`: + Use C99 types/limits. + - :c:macro:`Py_UNICODE_SIZE`: Use ``sizeof(wchar_t)`` directly. + - :c:macro:`Py_VA_COPY`: Use ``va_copy`` directly. + + The macro :c:macro:`Py_UNICODE_WIDE`, which was scheduled for removal, + is :term:`soft deprecated` instead. + + (Contributed by Petr Viktorin in :gh:`146175`.) + +* :c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated + since 3.15 and will be removed in 3.20. + (Contributed by Sergey B Kirpichev in :gh:`141004`.) + + +.. Add C API deprecations above alphabetically, not here at the end. + + +Build changes +============= + +* Removed implicit fallback to the bundled copy of the ``libmpdec`` library. + Now this should be explicitly enabled with :option:`!--with-system-libmpdec` + set to ``no`` or with :option:`!--without-system-libmpdec`. + (Contributed by Sergey B Kirpichev in :gh:`115119`.) + +* The new configure option :option:`--with-missing-stdlib-config=FILE` allows + distributors to pass a `JSON `_ + configuration file containing custom error messages for :term:`standard library` + modules that are missing or packaged separately. + (Contributed by Stan Ulbrych and Petr Viktorin in :gh:`139707`.) + +* The new configure option :option:`--with-pymalloc-hugepages` enables huge + page support for :ref:`pymalloc ` arenas. When enabled, arena size + increases to 2 MiB and allocation uses ``MAP_HUGETLB`` (Linux) or + ``MEM_LARGE_PAGES`` (Windows) with automatic fallback to regular pages. + On Windows, use ``build.bat --pymalloc-hugepages``. + At runtime, huge pages must be explicitly enabled by setting the + :envvar:`PYTHON_PYMALLOC_HUGEPAGES` environment variable to ``1``. + +* Annotating anonymous mmap usage is now supported if Linux kernel supports + :manpage:`PR_SET_VMA_ANON_NAME ` (Linux 5.17 or newer). + Annotations are visible in ``/proc//maps`` if the kernel supports the feature + and :option:`-X dev <-X>` is passed to the Python or Python is built in :ref:`debug mode `. + (Contributed by Donghee Na in :gh:`141770`.) + +* CPython is now built with frame pointers enabled by default + (:pep:`831`). Pass :option:`--without-frame-pointers` to opt out. + + Authors of C extensions and native libraries built with custom build + systems should ensure the unwind chain is intact. + This is usually done by adding ``-fno-omit-frame-pointer`` and + similar flags to ``CFLAGS``. See :option:`--without-frame-pointers` + documentation for the specific flags Python uses. + + (Contributed by Pablo Galindo Salgado and Savannah Ostrowski in :gh:`149201`.) + +.. _whatsnew315-windows-tail-calling-interpreter: + +* 64-bit builds using Visual Studio 2026 (MSVC 18) may now use the new + :ref:`tail-calling interpreter `. + Results on Visual Studio 18.1.1 report between + `15-20% `__ + speedup on the geometric mean of pyperformance on Windows x86-64 over + the switch-case interpreter on an AMD Ryzen 7 5800X. We have + observed speedups ranging from 14% for large pure-Python libraries + to 40% for long-running small pure-Python scripts on Windows. + This was made possible by a new feature introduced in MSVC 18, + which the official Windows 64-bit binaries on python.org__ now use. + (Contributed by Chris Eibl, Ken Jin, and Brandt Bucher in :gh:`143068`. + Special thanks to Steve Dower, and the MSVC team including Hulon Jenkins.) + + __ https://www.python.org/downloads/windows/ + + +Porting to Python 3.15 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + +* :class:`sqlite3.Connection` APIs have been cleaned up. + + * All parameters of :func:`sqlite3.connect` except *database* are now keyword-only. + * The first three parameters of methods :meth:`~sqlite3.Connection.create_function` + and :meth:`~sqlite3.Connection.create_aggregate` are now positional-only. + * The first parameter of methods :meth:`~sqlite3.Connection.set_authorizer`, + :meth:`~sqlite3.Connection.set_progress_handler` and + :meth:`~sqlite3.Connection.set_trace_callback` is now positional-only. + + (Contributed by Serhiy Storchaka in :gh:`133595`.) + +* :data:`resource.RLIM_INFINITY` is now always positive. + Passing a negative integer value that corresponded to its old value + (such as ``-1`` or ``-3``, depending on platform) to + :func:`resource.setrlimit` and :func:`resource.prlimit` is now deprecated. + (Contributed by Serhiy Storchaka in :gh:`137044`.) + +* :meth:`mmap.mmap.resize` has been removed on platforms that don't support the + underlying syscall, instead of raising a :exc:`SystemError`. + +* A resource warning is now emitted for an unclosed + :func:`xml.etree.ElementTree.iterparse` iterator if it opened a file. + Use its :meth:`!close` method or the :func:`contextlib.closing` context + manager to close it. + (Contributed by Osama Abdelkader and Serhiy Storchaka in :gh:`140601`.) + +* If a short option and a single-dash long option are passed to + :meth:`argparse.ArgumentParser.add_argument`, *dest* is now inferred from + the single-dash long option. For example, in ``add_argument('-f', '-foo')``, + *dest* is now ``'foo'`` instead of ``'f'``. + Pass an explicit *dest* argument to preserve the old behavior. + (Contributed by Serhiy Storchaka in :gh:`138697`.) + +* Padding of input no longer required in :func:`base64.urlsafe_b64decode`. + Pass a new argument ``padded=True`` or use :func:`base64.b64decode` + with argument ``altchars=b'-_'`` (this works with older Python versions) + to make padding required. + (Contributed by Serhiy Storchaka in :gh:`73613`.) + +* Since :meth:`unittest.TestCase.assertWarns` and + :meth:`unittest.TestCase.assertWarnsRegex` no longer swallow warnings that + do not match the specified category or regex, your tests may start leaking + some warnings that were previously masked. + Use warning filters to silence them or additional :meth:`!assertWarns*` + to catch and check them. + (Contributed by Serhiy Storchaka in :gh:`143231`.) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst new file mode 100644 index 000000000000000..ec8e367d938ddb9 --- /dev/null +++ b/Doc/whatsnew/3.16.rst @@ -0,0 +1,376 @@ + +**************************** + What's new in Python 3.16 +**************************** + +:Editor: TBD + +.. Rules for maintenance: + + * Anyone can add text to this document. Do not spend very much time + on the wording of your changes, because your text will probably + get rewritten to some degree. + + * The maintainer will go through Misc/NEWS periodically and add + changes; it's therefore more important to add your changes to + Misc/NEWS than to this file. + + * This is not a complete list of every single change; completeness + is the purpose of Misc/NEWS. Some changes I consider too small + or esoteric to include. If such a change is added to the text, + I'll just remove it. (This is another reason you shouldn't spend + too much time on writing your addition.) + + * If you want to draw your new text to the attention of the + maintainer, add 'XXX' to the beginning of the paragraph or + section. + + * It's OK to just add a fragmentary note about a change. For + example: "XXX Describe the transmogrify() function added to the + socket module." The maintainer will research the change and + write the necessary text. + + * You can comment out your additions if you like, but it's not + necessary (especially when a final release is some months away). + + * Credit the author of a patch or bugfix. Just the name is + sufficient; the e-mail address isn't necessary. + + * It's helpful to add the issue number as a comment: + + XXX Describe the transmogrify() function added to the socket + module. + (Contributed by P.Y. Developer in :gh:`12345`.) + + This saves the maintainer the effort of going through the VCS log + when researching a change. + +This article explains the new features in Python 3.16, compared to 3.15. + +For full details, see the :ref:`changelog `. + +.. note:: + + Prerelease users should be aware that this document is currently in draft + form. It will be updated substantially as Python 3.16 moves towards release, + so it's worth checking back even after reading earlier versions. + + +Summary --- release highlights +============================== + +.. This section singles out the most important changes in Python 3.16. + Brevity is key. + + +.. PEP-sized items next. + + + +New features +============ + + + +Other language changes +====================== + + + +New modules +=========== + +* None yet. + + +Improved modules +================ + +curses +------ + +* Add :func:`curses.nofilter`, which undoes the effect of :func:`curses.filter`. + (Contributed by Serhiy Storchaka in :gh:`151744`.) + +gzip +---- + +* :func:`gzip.open` now accepts an optional argument ``mtime`` + which is passed on to the constructor of the :class:`~gzip.GzipFile` class. + (Contributed by Marin Misur in :gh:`91372`.) + +lzma +---- + +* Add support of new BCJ filters ARM64 and RISC-V via + :const:`!lzma.FILTER_ARM64` and :const:`!lzma.FILTER_RISCV`. Note that the + new filters will work only if runtime library supports them. ARM64 filter + requires ``lzma`` 5.4.0 or newer while RISC-V requires 5.6.0 or newer. + (Contributed by Chien Wong in :gh:`115988`.) + +logging +------- + +* :class:`~logging.handlers.TimedRotatingFileHandler` now uses the creation + time instead of the last modification time of an existing log file as + the basis for the first rotation after handler creation, if supported by + the OS and file system. + This allows it to be used in short-running programs that start and end + before the rotation interval expires. + (Contributed by Iván Márton and Serhiy Storchaka in :gh:`84649`.) + +math +---- + +* Added trigonometric functions that work in units of half turns, rather than + radians. The new functions :func:`math.acospi`, :func:`math.asinpi`, + :func:`math.atanpi`, and :func:`math.atan2pi` return half-turn angles. The + new functions :func:`math.cospi`, :func:`math.sinpi`, and :func:`math.tanpi` + take half-turn angle arguments. These functions are recommended by IEEE + 754-2019 and standardized in C23. + (Contributed by Jeff Epler in :gh:`150534`.) + +os +-- + +* Add :func:`os.pidfd_getfd` for duplicating a file descriptor from another + process via a pidfd. Available on Linux 5.6+. + (Contributed by Maurycy Pawłowski-Wieroński in :gh:`149464`.) + +shlex +----- + +* Add keyword-only parameter *force* to :func:`shlex.quote` to force quoting + a string, even if it is already safe for a shell without being quoted. + (Contributed by Jay Berry in :gh:`148846`.) + +tkinter +------- + +* Added new :class:`!tkinter.Text` methods :meth:`~tkinter.Text.edit_canundo` + and :meth:`~tkinter.Text.edit_canredo` which return whether an undo or redo + is possible. + (Contributed by Serhiy Storchaka in :gh:`151674`.) + +* Added new :class:`!tkinter.Text` methods :meth:`~tkinter.Text.sync` and + :meth:`~tkinter.Text.pendingsync` which control and report the + synchronization of the displayed view with the underlying text. + (Contributed by Serhiy Storchaka in :gh:`151675`.) + +xml +--- + +* Add support for multiple multi-byte encodings in the :mod:`XML parser + `: "cp932", "cp949", "cp950", "Big5","EUC-JP", + "GB2312", "GBK", "johab", and "Shift_JIS". + Add partial support (only BMP characters) for multi-byte encodings + "Big5-HKSCS", "EUC_JIS-2004", "EUC_JISX0213", "Shift_JIS-2004", + "Shift_JISX0213", "utf-8-sig" and non-standard aliases like "UTF8" + (without hyphen). + The parser now raises :exc:`ValueError` for known unsupported + multi-byte encodings such us "ISO-2022-JP" or "raw-unicode-escape" + instead of failing later, when encounter non-ASCII data. + (Contributed by Serhiy Storchaka in :gh:`62259`.) + +zipfile +------- + +* Add :meth:`ZipFile.remove() ` to remove a member + from an archive's central directory, and + :meth:`ZipFile.repack() ` to reclaim the space used + by the local file entries of removed members. + (Contributed by Danny Lin in :gh:`51067`.) + +.. Add improved modules above alphabetically, not here at the end. + +Optimizations +============= + +module_name +----------- + +* TODO + + + +Removed +======= + +annotationlib +------------- + +* The :meth:`!annotationlib.ForwardRef._evaluate` method + which has been deprecated since Python 3.14. + Use :meth:`annotationlib.ForwardRef.evaluate` + or :func:`typing.evaluate_forward_ref` instead. + +array +----- + +* The ``'u'`` format code (:c:type:`wchar_t`) which has been deprecated in + documentation since Python 3.3 and at runtime since Python 3.13. + Use ``'w'`` format code instead (:c:type:`Py_UCS4`, always 4 bytes). + +asyncio +------- + +* The :func:`!asyncio.iscoroutinefunction` + which has been deprecated since Python 3.14. + Use :func:`inspect.iscoroutinefunction` instead. + +functools +--------- + +* Calling the Python implementation of :func:`functools.reduce` with *function* + or *sequence* as keyword arguments has been deprecated since Python 3.14. + +logging +------- + +* Support for custom logging handlers with the *strm* argument is deprecated + and scheduled for removal in Python 3.16. Define handlers with the *stream* + argument instead. + +mimetypes +--------- + +* Valid extensions start with a '.' or are empty for + :meth:`mimetypes.MimeTypes.add_type`. + Undotted extensions now raise a :exc:`ValueError`. + +shutil +------ + +* The :exc:`!ExecError` exception which has been deprecated since Python 3.14. + It has not been used by any function in :mod:`!shutil` since Python 3.4. + (Contributed by Stan Ulbrych in :gh:`149567`.) + +symtable +-------- + +* The :meth:`!symtable.Class.get_methods` method + which has been deprecated since Python 3.14. + +sys +--- + +* The :func:`!_enablelegacywindowsfsencoding` function + which has been deprecated since Python 3.13. + Use the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable instead. + (Contributed by Stan Ulbrych in :gh:`149595`.) + +sysconfig +--------- + +* The :func:`!sysconfig.expand_makefile_vars` function + which has been deprecated since Python 3.14. + Use the ``vars`` argument of :func:`sysconfig.get_paths` instead. + (Contributed by Stan Ulbrych in :gh:`149499`.) + +tarfile +------- + +* The undocumented and unused :attr:`!tarfile.TarFile.tarfile` attribute + has been deprecated since Python 3.13. + +.. Add removals above alphabetically, not here at the end. + + +Deprecated +========== + +* :mod:`abc` + + * Soft-deprecated since Python 3.3 :class:`abc.abstractclassmethod`, + :class:`abc.abstractstaticmethod`, and :class:`abc.abstractproperty` + now raise a :exc:`DeprecationWarning`. + These classes will be removed in Python 3.21, instead + use :func:`abc.abstractmethod` with :func:`classmethod`, + :func:`staticmethod`, and :class:`property` respectively. + +* :mod:`ast`: + + * Classes ``slice``, ``Index``, ``ExtSlice``, ``Suite``, ``Param``, + ``AugLoad`` and ``AugStore``, deprecated since Python 3.9, are no longer + imported by ``from ast import *`` and issue a deprecation warning on + use. The classes are slated for removal in Python 3.21. These types are not + generated by the parser or accepted by the code generator. + * The ``dims`` property of ``ast.Tuple`` objects, deprecated since Python + 3.9, now issues a deprecation warning on use. This property is slated for + removal in 3.21. Use ``ast.Tuple.elts`` instead. + +.. Add deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/pending-removal-in-3.17.rst + +.. include:: ../deprecations/pending-removal-in-3.18.rst + +.. include:: ../deprecations/pending-removal-in-3.19.rst + +.. include:: ../deprecations/pending-removal-in-3.20.rst + +.. include:: ../deprecations/pending-removal-in-3.21.rst + +.. include:: ../deprecations/pending-removal-in-future.rst + + +Porting to Python 3.16 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + + +Build changes +============= + +* Remove the bundled copy of the libmpdec_ decimal library from the CPython source tree + to simplify maintenence and updates. The :mod:`decimal` module will now + unconditionally use the system's libmpdec decimal library. Also remove the + now unused :option:`!--with-system-libmpdec` :program:`configure` flag. + This change has no impact on binary releases of Python, which have been + built against a separate copy of libmpdec for the past several releases. + + (Contributed by Sergey B Kirpichev in :gh:`115119`.) + + .. _libmpdec: https://www.bytereef.org/mpdecimal/ + +* Add a :option:`--with-build-details-suffix` configure flag to allow + Linux distributions that co-install multiple versions of Python in the + same tree to avoid ``build-details.json`` clashes. + + (Contributed by Stefano Rivera in :gh:`131372`.) + + +C API changes +============= + +New features +------------ + +* TODO + +Porting to Python 3.16 +---------------------- + +* TODO + +Deprecated C APIs +----------------- + +* :c:func:`PyGen_New`, :c:func:`PyGen_NewWithQualName`, :c:func:`PyCoro_New`, + and :c:func:`PyAsyncGen_New` are deprecated. + They are scheduled for removal in 3.18. + +.. Add C API deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/c-api-pending-removal-in-3.18.rst + +.. include:: ../deprecations/c-api-pending-removal-in-3.19.rst + +.. include:: ../deprecations/c-api-pending-removal-in-3.20.rst + +.. include:: ../deprecations/c-api-pending-removal-in-future.rst + +Removed C APIs +-------------- diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 7104904c956a7a9..c61eb659ccc0f62 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -312,7 +312,7 @@ cluttering source directories, the *pyc* files are now collected in a Aside from the filenames and target directories, the new scheme has a few aspects that are visible to the programmer: -* Imported modules now have a :attr:`~module.__cached__` attribute which stores +* Imported modules now have a ``__cached__`` attribute which stores the name of the actual file that was imported: >>> import collections @@ -458,7 +458,7 @@ Some smaller changes made to the core Python language are: :class:`~collections.defaultdict`, :class:`~shelve.Shelf`, :class:`~configparser.ConfigParser`, or :mod:`dbm`. It is also useful with custom :class:`dict` subclasses that normalize keys before look-up or that - supply a :meth:`__missing__` method for unknown keys:: + supply a :meth:`~object.__missing__` method for unknown keys:: >>> import shelve >>> d = shelve.open('tmp.shl') @@ -756,7 +756,7 @@ functools --------- * The :mod:`functools` module includes a new decorator for caching function - calls. :func:`functools.lru_cache` can save repeated queries to an external + calls. :deco:`functools.lru_cache` can save repeated queries to an external resource whenever the results are expected to be the same. For example, adding a caching decorator to a database query function can save @@ -789,7 +789,7 @@ functools `_\, :issue:`10586`, and :issue:`10593`.) -* The :func:`functools.wraps` decorator now adds a :attr:`__wrapped__` attribute +* The :deco:`functools.wraps` decorator now adds a :attr:`__wrapped__` attribute pointing to the original callable function. This allows wrapped functions to be introspected. It also copies :attr:`~function.__annotations__` if defined. And now it also gracefully skips over missing attributes such as @@ -805,11 +805,11 @@ functools :issue:`8814`.) * To help write classes with rich comparison methods, a new decorator - :func:`functools.total_ordering` will use existing equality and inequality + :deco:`functools.total_ordering` will use existing equality and inequality methods to fill in the remaining methods. For example, supplying *__eq__* and *__lt__* will enable - :func:`~functools.total_ordering` to fill-in *__le__*, *__gt__* and *__ge__*:: + :deco:`~functools.total_ordering` to fill-in *__le__*, *__gt__* and *__ge__*:: @total_ordering class Student: @@ -992,12 +992,12 @@ datetime and time offset and timezone name. This makes it easier to create timezone-aware datetime objects:: - >>> from datetime import datetime, timezone + >>> import datetime as dt - >>> datetime.now(timezone.utc) + >>> dt.datetime.now(dt.timezone.utc) datetime.datetime(2010, 12, 8, 21, 4, 2, 923754, tzinfo=datetime.timezone.utc) - >>> datetime.strptime("01/01/2000 12:00 +0000", "%m/%d/%Y %H:%M %z") + >>> dt.datetime.strptime("01/01/2000 12:00 +0000", "%m/%d/%Y %H:%M %z") datetime.datetime(2000, 1, 1, 12, 0, tzinfo=datetime.timezone.utc) * Also, :class:`~datetime.timedelta` objects can now be multiplied by @@ -1097,11 +1097,11 @@ logarithm of the gamma function: abc --- -The :mod:`abc` module now supports :func:`~abc.abstractclassmethod` and -:func:`~abc.abstractstaticmethod`. +The :mod:`abc` module now supports :deco:`~abc.abstractclassmethod` and +:deco:`~abc.abstractstaticmethod`. These tools make it possible to define an :term:`abstract base class` that -requires a particular :func:`classmethod` or :func:`staticmethod` to be +requires a particular :deco:`classmethod` or :deco:`staticmethod` to be implemented:: class Temperature(metaclass=abc.ABCMeta): @@ -1155,7 +1155,7 @@ self-reference by displaying "..." in the recursive part of the representation string. To help write such :meth:`~object.__repr__` methods, the :mod:`reprlib` module has a new -decorator, :func:`~reprlib.recursive_repr`, for detecting recursive calls to +decorator, :deco:`~reprlib.recursive_repr`, for detecting recursive calls to :meth:`!__repr__` and substituting a placeholder string instead:: >>> class MyList(list): @@ -1241,7 +1241,7 @@ There is a new and slightly mind-blowing tool :term:`context manager` that does double duty as a function decorator. As a convenience, this new functionality is used by -:func:`~contextlib.contextmanager` so that no extra effort is needed to support +:deco:`~contextlib.contextmanager` so that no extra effort is needed to support both roles. The basic idea is that both context managers and function decorators can be used @@ -1253,7 +1253,7 @@ write a pre-action or post-action wrapper that can be used in either role. For example, it is sometimes useful to wrap functions or groups of statements with a logger that can track the time of entry and time of exit. Rather than writing both a function decorator and a context manager for the task, the -:func:`~contextlib.contextmanager` provides both capabilities in a single +:deco:`~contextlib.contextmanager` provides both capabilities in a single definition:: from contextlib import contextmanager diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 89fd68686454e27..02fd264e53e1b06 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -331,7 +331,7 @@ simplified and finer-grained. You don't have to worry anymore about choosing the appropriate exception type between :exc:`OSError`, :exc:`IOError`, :exc:`EnvironmentError`, -:exc:`WindowsError`, :exc:`mmap.error`, :exc:`socket.error` or +:exc:`WindowsError`, :exc:`!mmap.error`, :exc:`socket.error` or :exc:`select.error`. All these exception types are now only one: :exc:`OSError`. The other names are kept as aliases for compatibility reasons. @@ -805,7 +805,7 @@ Some smaller changes made to the core Python language are: * New methods have been added to :class:`list` and :class:`bytearray`: ``copy()`` and ``clear()`` (:issue:`10516`). Consequently, :class:`~collections.abc.MutableSequence` now also defines a - :meth:`~collections.abc.MutableSequence.clear` method (:issue:`11388`). + :meth:`!clear` method (:issue:`11388`). * Raw bytes literals can now be written ``rb"..."`` as well as ``br"..."``. @@ -869,10 +869,10 @@ faulthandler This new debug module :mod:`faulthandler` contains functions to dump Python tracebacks explicitly, on a fault (a crash like a segmentation fault), after a timeout, or on a user signal. Call :func:`faulthandler.enable` to install fault handlers for the -:const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS`, and -:const:`SIGILL` signals. You can also enable them at startup by setting the -:envvar:`PYTHONFAULTHANDLER` environment variable or by using :option:`-X` -``faulthandler`` command line option. +:const:`~signal.SIGSEGV`, :const:`~signal.SIGFPE`, :const:`~signal.SIGABRT`, +:const:`~signal.SIGBUS`, and :const:`~signal.SIGILL` signals. +You can also enable them at startup by setting the :envvar:`PYTHONFAULTHANDLER` +environment variable or by using :option:`-X` ``faulthandler`` command line option. Example of a segmentation fault on Linux: @@ -916,15 +916,15 @@ abc Improved support for abstract base classes containing descriptors composed with abstract methods. The recommended approach to declaring abstract descriptors is -now to provide :attr:`__isabstractmethod__` as a dynamically updated +now to provide :attr:`!__isabstractmethod__` as a dynamically updated property. The built-in descriptors have been updated accordingly. -* :class:`abc.abstractproperty` has been deprecated, use :class:`property` - with :func:`abc.abstractmethod` instead. -* :class:`abc.abstractclassmethod` has been deprecated, use - :class:`classmethod` with :func:`abc.abstractmethod` instead. -* :class:`abc.abstractstaticmethod` has been deprecated, use - :class:`staticmethod` with :func:`abc.abstractmethod` instead. +* :deco:`abc.abstractproperty` has been deprecated, use :deco:`property` + with :deco:`abc.abstractmethod` instead. +* :deco:`abc.abstractclassmethod` has been deprecated, use + :deco:`classmethod` with :deco:`abc.abstractmethod` instead. +* :deco:`abc.abstractstaticmethod` has been deprecated, use + :deco:`staticmethod` with :deco:`abc.abstractmethod` instead. (Contributed by Darren Dale in :issue:`11610`.) @@ -979,7 +979,7 @@ new features have been added: (Contributed by Nir Aides in :issue:`1625`.) * :class:`bz2.BZ2File` now implements all of the :class:`io.BufferedIOBase` API, - except for the :meth:`detach` and :meth:`truncate` methods. + except for the :meth:`!detach` and :meth:`!truncate` methods. codecs @@ -1064,7 +1064,7 @@ curses * If the :mod:`curses` module is linked to the ncursesw library, use Unicode functions when Unicode strings or characters are passed (e.g. - :c:func:`waddwstr`), and bytes functions otherwise (e.g. :c:func:`waddstr`). + :c:func:`!waddwstr`), and bytes functions otherwise (e.g. :c:func:`!waddstr`). * Use the locale encoding instead of ``utf-8`` to encode Unicode strings. * :class:`curses.window` has a new :attr:`curses.window.encoding` attribute. * The :class:`curses.window` class has a new :meth:`~curses.window.get_wch` @@ -1137,15 +1137,15 @@ API changes * The C module has the following context limits, depending on the machine architecture: - +-------------------+----------------+-------------------------+ - | | 32-bit | 64-bit | - +===================+================+=========================+ - | :const:`MAX_PREC` | ``425000000`` | ``999999999999999999`` | - +-------------------+----------------+-------------------------+ - | :const:`MAX_EMAX` | ``425000000`` | ``999999999999999999`` | - +-------------------+----------------+-------------------------+ - | :const:`MIN_EMIN` | ``-425000000`` | ``-999999999999999999`` | - +-------------------+----------------+-------------------------+ + +----------------------------+----------------+-------------------------+ + | | 32-bit | 64-bit | + +============================+================+=========================+ + | :const:`~decimal.MAX_PREC` | ``425000000`` | ``999999999999999999`` | + +----------------------------+----------------+-------------------------+ + | :const:`~decimal.MAX_EMAX` | ``425000000`` | ``999999999999999999`` | + +----------------------------+----------------+-------------------------+ + | :const:`~decimal.MIN_EMIN` | ``-425000000`` | ``-999999999999999999`` | + +----------------------------+----------------+-------------------------+ * In the context templates (:const:`~decimal.DefaultContext`, :const:`~decimal.BasicContext` and :const:`~decimal.ExtendedContext`) @@ -1390,7 +1390,7 @@ ftplib functools --------- -The :func:`functools.lru_cache` decorator now accepts a ``typed`` keyword +The :deco:`functools.lru_cache` decorator now accepts a ``typed`` keyword argument (that defaults to ``False`` to ensure that it caches values of different types that compare equal in separate cache slots. (Contributed by Raymond Hettinger in :issue:`13227`.) @@ -1434,7 +1434,7 @@ html :class:`html.parser.HTMLParser` is now able to parse broken markup without raising errors, therefore the *strict* argument of the constructor and the -:exc:`~html.parser.HTMLParseError` exception are now deprecated. +:exc:`!HTMLParseError` exception are now deprecated. The ability to parse broken markup is the result of a number of bug fixes that are also available on the latest bug fix releases of Python 2.7/3.2. (Contributed by Ezio Melotti in :issue:`15114`, and :issue:`14538`, @@ -1486,7 +1486,7 @@ already exists. It is based on the C11 'x' mode to fopen(). The constructor of the :class:`~io.TextIOWrapper` class has a new *write_through* optional argument. If *write_through* is ``True``, calls to -:meth:`~io.TextIOWrapper.write` are guaranteed not to be buffered: any data +:meth:`!write` are guaranteed not to be buffered: any data written on the :class:`~io.TextIOWrapper` object is immediately handled to its underlying binary buffer. @@ -1504,7 +1504,7 @@ logging The :func:`~logging.basicConfig` function now supports an optional ``handlers`` argument taking an iterable of handlers to be added to the root logger. -A class level attribute :attr:`~logging.handlers.SysLogHandler.append_nul` has +A class level attribute :attr:`!append_nul` has been added to :class:`~logging.handlers.SysLogHandler` to allow control of the appending of the ``NUL`` (``\000``) byte to syslog records, since for some daemons it is required while for others it is passed through to the log. @@ -1536,8 +1536,8 @@ The new :func:`multiprocessing.connection.wait` function allows polling multiple objects (such as connections, sockets and pipes) with a timeout. (Contributed by Richard Oudkerk in :issue:`12328`.) -:class:`multiprocessing.Connection` objects can now be transferred over -multiprocessing connections. +:class:`multiprocessing.connection.Connection` objects can now be transferred +over multiprocessing connections. (Contributed by Richard Oudkerk in :issue:`4892`.) :class:`multiprocessing.Process` now accepts a ``daemon`` keyword argument @@ -1611,7 +1611,7 @@ os :func:`~os.rename`, :func:`~os.replace`, :func:`~os.rmdir`, :func:`~os.stat`, :func:`~os.symlink`, :func:`~os.unlink`, :func:`~os.utime`. Platform support for using these parameters can be checked via the sets - :data:`os.supports_dir_fd` and :data:`os.supports_follows_symlinks`. + :data:`os.supports_dir_fd` and :data:`os.supports_follow_symlinks`. - The following functions now support a file descriptor for their path argument: :func:`~os.chdir`, :func:`~os.chmod`, :func:`~os.chown`, @@ -1698,7 +1698,7 @@ os :const:`~os.RTLD_NOLOAD`, and :const:`~os.RTLD_DEEPBIND` are available on platforms that support them. These are for use with the :func:`sys.setdlopenflags` function, and supersede the similar constants - defined in :mod:`ctypes` and :mod:`DLFCN`. (Contributed by Victor Stinner + defined in :mod:`ctypes` and :mod:`!DLFCN`. (Contributed by Victor Stinner in :issue:`13226`.) * :func:`os.symlink` now accepts (and ignores) the ``target_is_directory`` @@ -1728,8 +1728,8 @@ reduction functions to be set. pydoc ----- -The Tk GUI and the :func:`~pydoc.serve` function have been removed from the -:mod:`pydoc` module: ``pydoc -g`` and :func:`~pydoc.serve` have been deprecated +The Tk GUI and the :func:`!serve` function have been removed from the +:mod:`pydoc` module: ``pydoc -g`` and :func:`!serve` have been deprecated in Python 3.2. @@ -1931,7 +1931,7 @@ ssl * :func:`~ssl.RAND_bytes`: generate cryptographically strong pseudo-random bytes. - * :func:`~ssl.RAND_pseudo_bytes`: generate pseudo-random bytes. + * :func:`!RAND_pseudo_bytes`: generate pseudo-random bytes. (Contributed by Victor Stinner in :issue:`12049`.) @@ -2020,8 +2020,7 @@ tarfile tempfile -------- -:class:`tempfile.SpooledTemporaryFile`\'s -:meth:`~tempfile.SpooledTemporaryFile.truncate` method now accepts +:class:`tempfile.SpooledTemporaryFile`\'s :meth:`!truncate` method now accepts a ``size`` parameter. (Contributed by Ryan Kelly in :issue:`9957`.) @@ -2129,7 +2128,7 @@ xml.etree.ElementTree The :mod:`xml.etree.ElementTree` module now imports its C accelerator by default; there is no longer a need to explicitly import -:mod:`xml.etree.cElementTree` (this module stays for backwards compatibility, +:mod:`!xml.etree.cElementTree` (this module stays for backwards compatibility, but is now deprecated). In addition, the ``iter`` family of methods of :class:`~xml.etree.ElementTree.Element` has been optimized (rewritten in C). The module's documentation has also been greatly improved with added examples @@ -2197,7 +2196,7 @@ Changes to Python's build process and to the C API include: * :c:func:`PyUnicode_AsUCS4`, :c:func:`PyUnicode_AsUCS4Copy` * :c:macro:`PyUnicode_DATA`, :c:macro:`PyUnicode_1BYTE_DATA`, :c:macro:`PyUnicode_2BYTE_DATA`, :c:macro:`PyUnicode_4BYTE_DATA` - * :c:macro:`PyUnicode_KIND` with :c:enum:`PyUnicode_Kind` enum: + * :c:macro:`PyUnicode_KIND` with :c:enum:`!PyUnicode_Kind` enum: :c:data:`!PyUnicode_WCHAR_KIND`, :c:data:`PyUnicode_1BYTE_KIND`, :c:data:`PyUnicode_2BYTE_KIND`, :c:data:`PyUnicode_4BYTE_KIND` * :c:macro:`PyUnicode_READ`, :c:macro:`PyUnicode_READ_CHAR`, :c:macro:`PyUnicode_WRITE` @@ -2232,25 +2231,25 @@ Deprecated Python modules, functions and methods (``utf-32-le`` or ``utf-32-be``) * :meth:`ftplib.FTP.nlst` and :meth:`ftplib.FTP.dir`: use :meth:`ftplib.FTP.mlsd` -* :func:`platform.popen`: use the :mod:`subprocess` module. Check especially +* :func:`!platform.popen`: use the :mod:`subprocess` module. Check especially the :ref:`subprocess-replacements` section (:issue:`11377`). * :issue:`13374`: The Windows bytes API has been deprecated in the :mod:`os` module. Use Unicode filenames, instead of bytes filenames, to not depend on the ANSI code page anymore and to support any filename. -* :issue:`13988`: The :mod:`xml.etree.cElementTree` module is deprecated. The +* :issue:`13988`: The :mod:`!xml.etree.cElementTree` module is deprecated. The accelerator is used automatically whenever available. -* The behaviour of :func:`time.clock` depends on the platform: use the new +* The behaviour of :func:`!time.clock` depends on the platform: use the new :func:`time.perf_counter` or :func:`time.process_time` function instead, depending on your requirements, to have a well defined behaviour. -* The :func:`os.stat_float_times` function is deprecated. +* The :func:`!os.stat_float_times` function is deprecated. * :mod:`abc` module: - * :class:`abc.abstractproperty` has been deprecated, use :class:`property` - with :func:`abc.abstractmethod` instead. - * :class:`abc.abstractclassmethod` has been deprecated, use - :class:`classmethod` with :func:`abc.abstractmethod` instead. - * :class:`abc.abstractstaticmethod` has been deprecated, use - :class:`staticmethod` with :func:`abc.abstractmethod` instead. + * :deco:`abc.abstractproperty` has been deprecated, use :deco:`property` + with :deco:`abc.abstractmethod` instead. + * :deco:`abc.abstractclassmethod` has been deprecated, use + :deco:`classmethod` with :deco:`abc.abstractmethod` instead. + * :deco:`abc.abstractstaticmethod` has been deprecated, use + :deco:`staticmethod` with :deco:`abc.abstractmethod` instead. * :mod:`importlib` package: @@ -2466,7 +2465,7 @@ Porting C code * In the course of changes to the buffer API the undocumented :c:member:`!smalltable` member of the :c:type:`Py_buffer` structure has been removed and the - layout of the :c:type:`PyMemoryViewObject` has changed. + layout of the :c:type:`!PyMemoryViewObject` has changed. All extensions relying on the relevant parts in ``memoryobject.h`` or ``object.h`` must be rebuilt. diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index e4f602a17ee9687..0067491c569cc05 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -2,7 +2,7 @@ What's New In Python 3.4 **************************** -:Author: R. David Murray (Editor) +:Author: \R. David Murray (Editor) .. Rules for maintenance: @@ -122,7 +122,7 @@ Significantly improved library modules: on Unix ` (:issue:`8713`). * :mod:`email` has a new submodule, :mod:`~email.contentmanager`, and a new :mod:`~email.message.Message` subclass - (:class:`~email.contentmanager.EmailMessage`) that :ref:`simplify MIME + (:class:`~email.message.EmailMessage`) that :ref:`simplify MIME handling ` (:issue:`18891`). * The :mod:`inspect` and :mod:`pydoc` modules are now capable of correct introspection of a much wider variety of callable objects, @@ -154,7 +154,7 @@ Security improvements: `. * All modules in the standard library that support SSL now support server certificate verification, including hostname matching - (:func:`ssl.match_hostname`) and CRLs (Certificate Revocation lists, see + (:func:`!ssl.match_hostname`) and CRLs (Certificate Revocation lists, see :func:`ssl.SSLContext.load_verify_locations`). CPython implementation improvements: @@ -739,7 +739,7 @@ these new components. In addition, a new application-friendly class :class:`~dis.Bytecode` provides an object-oriented API for inspecting bytecode in both in human-readable form and for iterating over instructions. The :class:`~dis.Bytecode` constructor -takes the same arguments that :func:`~dis.get_instruction` does (plus an +takes the same arguments that :func:`~dis.get_instructions` does (plus an optional *current_offset*), and the resulting object can be iterated to produce :class:`~dis.Instruction` objects. But it also has a :mod:`~dis.Bytecode.dis` method, equivalent to calling :mod:`~dis.dis` on the constructor argument, but @@ -858,7 +858,7 @@ to behave like normal instance methods when included in a class definition. .. _whatsnew-singledispatch: -The new :func:`~functools.singledispatch` decorator brings support for +The new :deco:`~functools.singledispatch` decorator brings support for single-dispatch generic functions to the Python standard library. Where object oriented programming focuses on grouping multiple operations on a common set of data into a class, a generic function focuses on grouping @@ -870,7 +870,7 @@ multiple implementations of an operation that allows it to work with :pep:`443` -- Single-dispatch generic functions PEP written and implemented by Łukasz Langa. -:func:`~functools.total_ordering` now supports a return value of +:deco:`~functools.total_ordering` now supports a return value of :data:`NotImplemented` from the underlying comparison function. (Contributed by Katie Miller in :issue:`10042`.) @@ -958,8 +958,9 @@ http optional additional *explain* parameter which can be used to provide an extended error description, overriding the hardcoded default if there is one. This extended error description will be formatted using the -:attr:`~http.server.HTTP.error_message_format` attribute and sent as the body -of the error response. (Contributed by Karl Cow in :issue:`12921`.) +:attr:`~http.server.BaseHTTPRequestHandler.error_message_format` attribute +and sent as the body of the error response. +(Contributed by Karl Cow in :issue:`12921`.) The :mod:`http.server` :ref:`command line interface ` now has a ``-b/--bind`` option that causes the server to listen on a specific address. @@ -1028,7 +1029,7 @@ information for modules, classes and functions. (Contributed by Claudiu Popa and Nick Coghlan in :issue:`18626`.) :func:`~inspect.unwrap` makes it easy to unravel wrapper function chains -created by :func:`functools.wraps` (and any other API that sets the +created by :deco:`functools.wraps` (and any other API that sets the ``__wrapped__`` attribute on a wrapper function). (Contributed by Daniel Urban, Aaron Iles and Nick Coghlan in :issue:`13266`.) @@ -1038,7 +1039,7 @@ As part of the implementation of the new :mod:`enum` module, the metaclasses. (Contributed by Ethan Furman in :issue:`18929` and :issue:`19030`.) -:func:`~inspect.getfullargspec` and :func:`~inspect.getargspec` +:func:`~inspect.getfullargspec` and :func:`!getargspec` now use the :func:`~inspect.signature` API. This allows them to support a much broader range of callables, including those with ``__signature__`` attributes, those with metadata provided by argument @@ -1221,7 +1222,7 @@ pickle :mod:`pickle` now supports (but does not use by default) a new pickle protocol, protocol 4. This new protocol addresses a number of issues that were present in previous protocols, such as the serialization of nested classes, very large -strings and containers, and classes whose :meth:`__new__` method takes +strings and containers, and classes whose :meth:`~object.__new__` method takes keyword-only arguments. It also provides some efficiency improvements. .. seealso:: @@ -1299,7 +1300,7 @@ affect the behaviour of :func:`help`. re -- -New :func:`~re.fullmatch` function and :meth:`.regex.fullmatch` method anchor +New :func:`~re.fullmatch` function and :meth:`.Pattern.fullmatch` method anchor the pattern at both ends of the string to match. This provides a way to be explicit about the goal of the match, which avoids a class of subtle bugs where ``$`` characters get lost during code changes or the addition of alternatives @@ -1519,7 +1520,7 @@ subprocess be used to provide the contents of ``stdin`` for the command that is run. (Contributed by Zack Weinberg in :issue:`16624`.) -:func:`~subprocess.getstatus` and :func:`~subprocess.getstatusoutput` now +:func:`~subprocess.getoutput` and :func:`~subprocess.getstatusoutput` now work on Windows. This change was actually inadvertently made in 3.3.4. (Contributed by Tim Golden in :issue:`10197`.) @@ -1535,7 +1536,7 @@ plain tuple. (Contributed by Claudiu Popa in :issue:`18901`.) called automatically at the end of the block. (Contributed by Serhiy Storchaka in :issue:`18878`.) -:meth:`.AU_write.setsampwidth` now supports 24 bit samples, thus adding +:meth:`!AU_write.setsampwidth` now supports 24 bit samples, thus adding support for writing 24 sample using the module. (Contributed by Serhiy Storchaka in :issue:`19261`.) @@ -1615,7 +1616,7 @@ A new :func:`~types.DynamicClassAttribute` descriptor provides a way to define an attribute that acts normally when looked up through an instance object, but which is routed to the *class* ``__getattr__`` when looked up through the class. This allows one to have properties active on a class, and have virtual -attributes on the class with the same name (see :mod:`Enum` for an example). +attributes on the class with the same name (see :mod:`enum` for an example). (Contributed by Ethan Furman in :issue:`19030`.) @@ -1709,7 +1710,7 @@ matching calls, which means an argument can now be matched by either position or name, instead of only by position. (Contributed by Antoine Pitrou in :issue:`17015`.) -:func:`~mock.mock_open` objects now have ``readline`` and ``readlines`` +:func:`~unittest.mock.mock_open` objects now have ``readline`` and ``readlines`` methods. (Contributed by Toshio Kuratomi in :issue:`17467`.) @@ -1729,8 +1730,8 @@ installed in the virtual environment. (Contributed by Nick Coghlan in wave ---- -The :meth:`~wave.getparams` method now returns a namedtuple rather than a -plain tuple. (Contributed by Claudiu Popa in :issue:`17487`.) +The :meth:`~wave.Wave_read.getparams` method now returns a namedtuple rather +than a plain tuple. (Contributed by Claudiu Popa in :issue:`17487`.) :meth:`wave.open` now supports the context management protocol. (Contributed by Claudiu Popa in :issue:`17616`.) @@ -1788,7 +1789,7 @@ example, this could be used to exclude test files from the archive. (Contributed by Christian Tismer in :issue:`19274`.) The *allowZip64* parameter to :class:`~zipfile.ZipFile` and -:class:`~zipfile.PyZipfile` is now ``True`` by default. (Contributed by +:class:`~zipfile.PyZipFile` is now ``True`` by default. (Contributed by William Mallard in :issue:`17201`.) @@ -1817,7 +1818,7 @@ PEP 442: Safe Object Finalization --------------------------------- :pep:`442` removes the current limitations and quirks of object finalization -in CPython. With it, objects with :meth:`__del__` methods, as well as +in CPython. With it, objects with :meth:`~object.__del__` methods, as well as generators with :keyword:`finally` clauses, can be finalized when they are part of a reference cycle. @@ -2043,7 +2044,7 @@ Significant Optimizations strings is now significantly faster. (Contributed by Victor Stinner and Antoine Pitrou in :issue:`15596`.) -* A performance issue in :meth:`io.FileIO.readall` has been solved. This +* A performance issue in :meth:`!io.FileIO.readall` has been solved. This particularly affects Windows, and significantly speeds up the case of piping significant amounts of data through :mod:`subprocess`. (Contributed by Richard Oudkerk in :issue:`15758`.) @@ -2085,10 +2086,10 @@ Deprecations in the Python API :meth:`!importlib.abc.PathEntryFinder.find_loader` and :meth:`!find_module` are replaced by :meth:`importlib.abc.PathEntryFinder.find_spec`; all of the :samp:`{xxx}Loader` ABC - ``load_module`` methods (:meth:`!importlib.abc.Loader.load_module`, - :meth:`!importlib.abc.InspectLoader.load_module`, - :meth:`!importlib.abc.FileLoader.load_module`, - :meth:`!importlib.abc.SourceLoader.load_module`) should no longer be + ``load_module`` methods (``importlib.abc.Loader.load_module``, + ``importlib.abc.InspectLoader.load_module``, + ``importlib.abc.FileLoader.load_module``, + ``importlib.abc.SourceLoader.load_module``) should no longer be implemented, instead loaders should implement an ``exec_module`` method (:meth:`importlib.abc.Loader.exec_module`, @@ -2103,7 +2104,7 @@ Deprecations in the Python API * The :mod:`!imp` module is pending deprecation. To keep compatibility with Python 2/3 code bases, the module's removal is currently not scheduled. -* The :mod:`formatter` module is pending deprecation and is slated for removal +* The :mod:`!formatter` module is pending deprecation and is slated for removal in Python 3.6. * ``MD5`` as the default *digestmod* for the :func:`hmac.new` function is @@ -2120,11 +2121,11 @@ Deprecations in the Python API * The *strict* argument of :class:`~html.parser.HTMLParser` is deprecated. -* The :mod:`plistlib` :func:`~plistlib.readPlist`, - :func:`~plistlib.writePlist`, :func:`~plistlib.readPlistFromBytes`, and - :func:`~plistlib.writePlistToBytes` functions are deprecated in favor of the +* The :mod:`plistlib` :func:`!readPlist`, + :func:`!writePlist`, :func:`!readPlistFromBytes`, and + :func:`!writePlistToBytes` functions are deprecated in favor of the corresponding new functions :func:`~plistlib.load`, :func:`~plistlib.dump`, - :func:`~plistlib.loads`, and :func:`~plistlib.dumps`. :func:`~plistlib.Data` + :func:`~plistlib.loads`, and :func:`~plistlib.dumps`. :func:`!Data` is deprecated in favor of just using the :class:`bytes` constructor. * The :mod:`sysconfig` key ``SO`` is deprecated, it has been replaced by @@ -2212,8 +2213,8 @@ removed: that do not have a __format__ method that handles it. See :issue:`7994` for background. -* :meth:`difflib.SequenceMatcher.isbjunk` and - :meth:`difflib.SequenceMatcher.isbpopular` were deprecated in 3.2, and have +* :meth:`!difflib.SequenceMatcher.isbjunk` and + :meth:`!difflib.SequenceMatcher.isbpopular` were deprecated in 3.2, and have now been removed: use ``x in sm.bjunk`` and ``x in sm.bpopular``, where *sm* is a :class:`~difflib.SequenceMatcher` object (:issue:`13248`). @@ -2280,7 +2281,7 @@ Changes in the Python API * :meth:`!importlib.util.module_for_loader` now sets ``__loader__`` and ``__package__`` unconditionally to properly support reloading. If this is not desired then you will need to set these attributes manually. You can use - :func:`importlib.util.module_to_load` for module management. + :func:`!importlib.util.module_to_load` for module management. * Import now resets relevant attributes (e.g. ``__name__``, ``__loader__``, ``__package__``, ``__file__``, ``__cached__``) unconditionally when reloading. @@ -2316,7 +2317,7 @@ Changes in the Python API wish to continue to ignore syntax or decoding issues, catch all three exceptions now. -* :func:`functools.update_wrapper` and :func:`functools.wraps` now correctly +* :func:`functools.update_wrapper` and :deco:`functools.wraps` now correctly set the ``__wrapped__`` attribute to the function being wrapped, even if that function also had its ``__wrapped__`` attribute set. This means ``__wrapped__`` attributes now correctly link a stack of decorated @@ -2428,7 +2429,7 @@ Changes in the Python API disallowed command forms didn't make any sense and are unlikely to be in use. * The :func:`re.split`, :func:`re.findall`, and :func:`re.sub` functions, and - the :meth:`~re.match.group` and :meth:`~re.match.groups` methods of + the :meth:`~re.Match.group` and :meth:`~re.Match.groups` methods of ``match`` objects now always return a *bytes* object when the string to be matched is a :term:`bytes-like object`. Previously the return type matched the input type, so if your code was depending on the return value diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index db3f1db3bd74ad5..24a2ffbefd0cfba 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -121,7 +121,7 @@ Significant improvements in the standard library: :ref:`better and significantly faster way ` of directory traversal. -* :func:`functools.lru_cache` has been mostly +* :deco:`functools.lru_cache` has been mostly :ref:`reimplemented in C `, yielding much better performance. @@ -181,7 +181,7 @@ Coroutine functions are declared using the new :keyword:`async def` syntax:: Inside a coroutine function, the new :keyword:`await` expression can be used to suspend coroutine execution until the result is available. Any object can be *awaited*, as long as it implements the :term:`awaitable` protocol by -defining the :meth:`__await__` method. +defining the :meth:`~object.__await__` method. PEP 492 also adds :keyword:`async for` statement for convenient iteration over asynchronous iterables. @@ -273,9 +273,10 @@ PEP 465 - A dedicated infix operator for matrix multiplication :pep:`465` adds the ``@`` infix operator for matrix multiplication. Currently, no builtin Python types implement the new operator, however, it -can be implemented by defining :meth:`__matmul__`, :meth:`__rmatmul__`, -and :meth:`__imatmul__` for regular, reflected, and in-place matrix -multiplication. The semantics of these methods is similar to that of +can be implemented by defining :meth:`~object.__matmul__`, +:meth:`~object.__rmatmul__`, and :meth:`~object.__imatmul__` for regular, +reflected, and in-place matrix multiplication. +The semantics of these methods is similar to that of methods defining other infix arithmetic operators. Matrix multiplication is a notably common operation in many fields of @@ -800,7 +801,7 @@ Notable changes in the :mod:`asyncio` module since Python 3.4.0: control. (Contributed by Victor Stinner.) -* The :func:`~asyncio.async` function is deprecated in favor of +* The :func:`!async` function is deprecated in favor of :func:`~asyncio.ensure_future`. (Contributed by Yury Selivanov.) @@ -905,10 +906,8 @@ collections The :class:`~collections.OrderedDict` class is now implemented in C, which makes it 4 to 100 times faster. (Contributed by Eric Snow in :issue:`16991`.) -:meth:`OrderedDict.items() `, -:meth:`OrderedDict.keys() `, -:meth:`OrderedDict.values() ` views now support -:func:`reversed` iteration. +:meth:`!OrderedDict.items`, :meth:`!OrderedDict.keys`, +and :meth:`!OrderedDict.values` views now support :func:`reversed` iteration. (Contributed by Serhiy Storchaka in :issue:`19505`.) The :class:`~collections.deque` class now defines @@ -928,7 +927,7 @@ Docstrings produced by :func:`~collections.namedtuple` can now be updated:: (Contributed by Berker Peksag in :issue:`24064`.) The :class:`~collections.UserString` class now implements the -:meth:`__getnewargs__`, :meth:`__rmod__`, :meth:`~str.casefold`, +:meth:`~object.__getnewargs__`, :meth:`~object.__rmod__`, :meth:`~str.casefold`, :meth:`~str.format_map`, :meth:`~str.isprintable`, and :meth:`~str.maketrans` methods to match the corresponding methods of :class:`str`. (Contributed by Joe Jevnik in :issue:`22189`.) @@ -937,7 +936,7 @@ methods to match the corresponding methods of :class:`str`. collections.abc --------------- -The :meth:`Sequence.index() ` method now +The :meth:`!Sequence.index` method now accepts *start* and *stop* arguments to match the corresponding methods of :class:`tuple`, :class:`list`, etc. (Contributed by Devin Jeanpierre in :issue:`23086`.) @@ -1045,8 +1044,8 @@ not just sequences. (Contributed by Serhiy Storchaka in :issue:`23171`.) curses ------ -The new :func:`~curses.update_lines_cols` function updates the :data:`LINES` -and :data:`COLS` module variables. This is useful for detecting +The new :func:`~curses.update_lines_cols` function updates the :data:`~curses.LINES` +and :data:`~curses.COLS` module variables. This is useful for detecting manual screen resizing. (Contributed by Arnon Yaari in :issue:`4254`.) @@ -1149,7 +1148,7 @@ functools .. _whatsnew-lrucache: -Most of the :func:`~functools.lru_cache` machinery is now implemented in C, making +Most of the :deco:`~functools.lru_cache` machinery is now implemented in C, making it significantly faster. (Contributed by Matt Joiner, Alexey Kachayev, and Serhiy Storchaka in :issue:`14373`.) @@ -1347,8 +1346,8 @@ network objects from existing addresses:: (Contributed by Peter Moody and Antoine Pitrou in :issue:`16531`.) -A new :attr:`~ipaddress.IPv4Network.reverse_pointer` attribute for the -:class:`~ipaddress.IPv4Network` and :class:`~ipaddress.IPv6Network` classes +A new :attr:`~ipaddress.IPv4Address.reverse_pointer` attribute for the +:class:`~ipaddress.IPv4Address` and :class:`~ipaddress.IPv6Address` classes returns the name of the reverse DNS PTR record:: >>> import ipaddress @@ -1451,7 +1450,7 @@ and :data:`~math.nan`. (Contributed by Mark Dickinson in :issue:`23185`.) A new function :func:`~math.isclose` provides a way to test for approximate equality. (Contributed by Chris Barker and Tal Einat in :issue:`24270`.) -A new :func:`~math.gcd` function has been added. The :func:`fractions.gcd` +A new :func:`~math.gcd` function has been added. The :func:`!fractions.gcd` function is now deprecated. (Contributed by Mark Dickinson and Serhiy Storchaka in :issue:`22486`.) @@ -1602,10 +1601,10 @@ The :func:`~re.sub` and :func:`~re.subn` functions now replace unmatched groups with empty strings instead of raising an exception. (Contributed by Serhiy Storchaka in :issue:`1519638`.) -The :class:`re.error` exceptions have new attributes, -:attr:`~re.error.msg`, :attr:`~re.error.pattern`, -:attr:`~re.error.pos`, :attr:`~re.error.lineno`, -and :attr:`~re.error.colno`, that provide better context +The :class:`re.error ` exceptions have new attributes, +:attr:`~re.PatternError.msg`, :attr:`~re.PatternError.pattern`, +:attr:`~re.PatternError.pos`, :attr:`~re.PatternError.lineno`, +and :attr:`~re.PatternError.colno`, that provide better context information about the error:: >>> re.compile(""" @@ -1794,10 +1793,10 @@ query the actual protocol version in use. (Contributed by Antoine Pitrou in :issue:`20421`.) The :class:`~ssl.SSLSocket` class now implements -a :meth:`SSLSocket.sendfile() ` method. +a :meth:`!SSLSocket.sendfile` method. (Contributed by Giampaolo Rodola' in :issue:`17552`.) -The :meth:`SSLSocket.send() ` method now raises either +The :meth:`!SSLSocket.send` method now raises either the :exc:`ssl.SSLWantReadError` or :exc:`ssl.SSLWantWriteError` exception on a non-blocking socket if the operation would block. Previously, it would return ``0``. (Contributed by Nikolaus Rath in :issue:`20951`.) @@ -1806,20 +1805,20 @@ The :func:`~ssl.cert_time_to_seconds` function now interprets the input time as UTC and not as local time, per :rfc:`5280`. Additionally, the return value is always an :class:`int`. (Contributed by Akira Li in :issue:`19940`.) -New :meth:`SSLObject.shared_ciphers() ` and +New :meth:`!SSLObject.shared_ciphers` and :meth:`SSLSocket.shared_ciphers() ` methods return the list of ciphers sent by the client during the handshake. (Contributed by Benjamin Peterson in :issue:`23186`.) The :meth:`SSLSocket.do_handshake() `, :meth:`SSLSocket.read() `, -:meth:`SSLSocket.shutdown() `, and +:meth:`!SSLSocket.shutdown`, and :meth:`SSLSocket.write() ` methods of the :class:`~ssl.SSLSocket` class no longer reset the socket timeout every time bytes are received or sent. The socket timeout is now the maximum total duration of the method. (Contributed by Victor Stinner in :issue:`23853`.) -The :func:`~ssl.match_hostname` function now supports matching of IP addresses. +The :func:`!match_hostname` function now supports matching of IP addresses. (Contributed by Antoine Pitrou in :issue:`23239`.) @@ -1863,10 +1862,10 @@ Examples:: sys --- -A new :func:`~sys.set_coroutine_wrapper` function allows setting a global +A new :func:`!set_coroutine_wrapper` function allows setting a global hook that will be called whenever a :term:`coroutine object ` is created by an :keyword:`async def` function. A corresponding -:func:`~sys.get_coroutine_wrapper` can be used to obtain a currently set +:func:`!get_coroutine_wrapper` can be used to obtain a currently set wrapper. Both functions are :term:`provisional `, and are intended for debugging purposes only. (Contributed by Yury Selivanov in :issue:`24017`.) @@ -2014,8 +2013,9 @@ The :class:`~unittest.mock.Mock` class has the following improvements: method to check if the mock object was called. (Contributed by Kushal Das in :issue:`21262`.) -The :class:`~unittest.mock.MagicMock` class now supports :meth:`__truediv__`, -:meth:`__divmod__` and :meth:`__matmul__` operators. +The :class:`~unittest.mock.MagicMock` class now supports +:meth:`~object.__truediv__`, :meth:`~object.__divmod__` +and :meth:`~object.__matmul__` operators. (Contributed by Johannes Baiter in :issue:`20968`, and Håkan Lövdahl in :issue:`23581` and :issue:`23568`.) @@ -2158,7 +2158,7 @@ improvement in some benchmarks. Objects from the :mod:`random` module now use 50% less memory on 64-bit builds. (Contributed by Serhiy Storchaka in :issue:`23488`.) -The :func:`property` getter calls are up to 25% faster. +The :deco:`property` getter calls are up to 25% faster. (Contributed by Joe Jevnik in :issue:`23910`.) Instantiation of :class:`fractions.Fraction` is now up to 30% faster. @@ -2290,10 +2290,10 @@ Windows XP is no longer supported by Microsoft, thus, per :PEP:`11`, CPython Deprecated Python modules, functions and methods ------------------------------------------------ -The :mod:`formatter` module has now graduated to full deprecation and is still +The :mod:`!formatter` module has now graduated to full deprecation and is still slated for removal in Python 3.6. -The :func:`asyncio.async` function is deprecated in favor of +The :func:`!asyncio.async` function is deprecated in favor of :func:`~asyncio.ensure_future`. The :mod:`!smtpd` module has in the past always decoded the DATA portion of @@ -2314,7 +2314,7 @@ Passing a format string as keyword argument *format_string* to the class has been deprecated. (Contributed by Serhiy Storchaka in :issue:`23671`.) -The :func:`platform.dist` and :func:`platform.linux_distribution` functions +The :func:`!platform.dist` and :func:`!platform.linux_distribution` functions are now deprecated. Linux distributions use too many different ways of describing themselves, so the functionality is left to a package. (Contributed by Vajrasky Kok and Berker Peksag in :issue:`1322`.) @@ -2324,11 +2324,11 @@ The previously undocumented ``from_function`` and ``from_builtin`` methods of :meth:`Signature.from_callable() ` method instead. (Contributed by Yury Selivanov in :issue:`24248`.) -The :func:`inspect.getargspec` function is deprecated and scheduled to be +The :func:`!inspect.getargspec` function is deprecated and scheduled to be removed in Python 3.6. (See :issue:`20438` for details.) The :mod:`inspect` :func:`~inspect.getfullargspec`, -:func:`~inspect.getcallargs`, and :func:`~inspect.formatargspec` functions are +:func:`~inspect.getcallargs`, and :func:`!formatargspec` functions are deprecated in favor of the :func:`inspect.signature` API. (Contributed by Yury Selivanov in :issue:`20438`.) @@ -2405,7 +2405,7 @@ Changes in the Python API error-prone and has been removed in Python 3.5. See :issue:`13936` for full details. -* The :meth:`ssl.SSLSocket.send` method now raises either +* The :meth:`!ssl.SSLSocket.send` method now raises either :exc:`ssl.SSLWantReadError` or :exc:`ssl.SSLWantWriteError` on a non-blocking socket if the operation would block. Previously, it would return ``0``. (Contributed by Nikolaus Rath in :issue:`20951`.) @@ -2440,12 +2440,12 @@ Changes in the Python API :mod:`http.client` and :mod:`http.server` remain available for backwards compatibility. (Contributed by Demian Brecht in :issue:`21793`.) -* When an import loader defines :meth:`importlib.machinery.Loader.exec_module` +* When an import loader defines :meth:`~importlib.abc.Loader.exec_module` it is now expected to also define - :meth:`~importlib.machinery.Loader.create_module` (raises a + :meth:`~importlib.abc.Loader.create_module` (raises a :exc:`DeprecationWarning` now, will be an error in Python 3.6). If the loader inherits from :class:`importlib.abc.Loader` then there is nothing to do, else - simply define :meth:`~importlib.machinery.Loader.create_module` to return + simply define :meth:`~importlib.abc.Loader.create_module` to return ``None``. (Contributed by Brett Cannon in :issue:`23014`.) * The :func:`re.split` function always ignored empty pattern matches, so the @@ -2488,7 +2488,7 @@ Changes in the Python API the POT-Creation-Date header. * The :mod:`smtplib` module now uses :data:`sys.stderr` instead of the previous - module-level :data:`stderr` variable for debug output. If your (test) + module-level :data:`!stderr` variable for debug output. If your (test) program depends on patching the module-level variable to capture the debug output, you will need to update it to capture sys.stderr instead. @@ -2514,11 +2514,11 @@ Changes in the C API -------------------- * The undocumented :c:member:`!format` member of the - (non-public) :c:type:`PyMemoryViewObject` structure has been removed. + (non-public) :c:type:`!PyMemoryViewObject` structure has been removed. All extensions relying on the relevant parts in ``memoryobject.h`` must be rebuilt. -* The :c:type:`PyMemAllocator` structure was renamed to +* The :c:type:`!PyMemAllocator` structure was renamed to :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. * Removed non-documented macro :c:macro:`!PyObject_REPR()` which leaked references. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 050c9103b00c98d..b0382dd8215bc00 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -517,7 +517,7 @@ Applications that do not use str to represent paths should use :func:`os.fsencode` and :func:`os.fsdecode` to ensure their bytes are correctly encoded. To revert to the previous behaviour, set :envvar:`PYTHONLEGACYWINDOWSFSENCODING` or call -:func:`sys._enablelegacywindowsfsencoding`. +:func:`!sys._enablelegacywindowsfsencoding`. See :pep:`529` for more information and discussion of code modifications that may be required. @@ -745,7 +745,7 @@ Some smaller changes made to the core Python language are: * It is now possible to set a :ref:`special method ` to ``None`` to indicate that the corresponding operation is not available. - For example, if a class sets :meth:`__iter__` to ``None``, the class + For example, if a class sets :meth:`~object.__iter__` to ``None``, the class is not iterable. (Contributed by Andrew Barnert and Ivan Levkivskyi in :issue:`25958`.) @@ -871,7 +871,7 @@ Notable changes in the :mod:`asyncio` module since Python 3.5.0 of the last iteration will be discarded. (Contributed by Guido van Rossum in :issue:`25593`.) -* :meth:`Future.set_exception ` +* :meth:`Future.set_exception ` will now raise :exc:`TypeError` when passed an instance of the :exc:`StopIteration` exception. (Contributed by Chris Angelico in :issue:`26221`.) @@ -925,7 +925,7 @@ added to represent sized iterable container classes. (Contributed by Ivan Levkivskyi, docs by Neil Girdhar in :issue:`27598`.) The new :class:`~collections.abc.Reversible` abstract base class represents -iterable classes that also provide the :meth:`__reversed__` method. +iterable classes that also provide the :meth:`~object.__reversed__` method. (Contributed by Ivan Levkivskyi in :issue:`25987`.) The new :class:`~collections.abc.AsyncGenerator` abstract base class represents @@ -971,7 +971,7 @@ datetime -------- The :class:`~datetime.datetime` and :class:`~datetime.time` classes have -the new :attr:`~time.fold` attribute used to disambiguate local time +the new :attr:`~datetime.time.fold` attribute used to disambiguate local time when necessary. Many functions in the :mod:`datetime` have been updated to support local time disambiguation. See :ref:`Local Time Disambiguation ` section for more @@ -1052,12 +1052,12 @@ enum ---- Two new enumeration base classes have been added to the :mod:`enum` module: -:class:`~enum.Flag` and :class:`~enum.IntFlags`. Both are used to define +:class:`~enum.Flag` and :class:`~enum.IntFlag`. Both are used to define constants that can be combined using the bitwise operators. (Contributed by Ethan Furman in :issue:`23591`.) Many standard library modules have been updated to use the -:class:`~enum.IntFlags` class for their constants. +:class:`~enum.IntFlag` class for their constants. The new :class:`enum.auto` value can be used to assign values to enum members automatically:: @@ -1224,7 +1224,7 @@ generator expression scopes as if they were positional-only parameters called ``implicit0``. (Contributed by Jelle Zijlstra in :issue:`19611`.) To reduce code churn when upgrading from Python 2.7 and the legacy -:func:`inspect.getargspec` API, the previously documented deprecation of +:func:`!inspect.getargspec` API, the previously documented deprecation of :func:`inspect.getfullargspec` has been reversed. While this function is convenient for single/source Python 2/3 code bases, the richer :func:`inspect.signature` interface remains the recommended approach for new @@ -1275,7 +1275,7 @@ See the summary of :ref:`PEP 519 ` for details on how the A new :meth:`~os.scandir.close` method allows explicitly closing a :func:`~os.scandir` iterator. The :func:`~os.scandir` iterator now -supports the :term:`context manager` protocol. If a :func:`scandir` +supports the :term:`context manager` protocol. If a :func:`!scandir` iterator is neither exhausted nor explicitly closed a :exc:`ResourceWarning` will be emitted in its destructor. (Contributed by Serhiy Storchaka in :issue:`25994`.) @@ -1434,7 +1434,7 @@ defined in :mod:`http.server`, :mod:`xmlrpc.server` and protocol. (Contributed by Aviv Palivoda in :issue:`26404`.) -The :attr:`~socketserver.StreamRequestHandler.wfile` attribute of +The :attr:`wfile ` attribute of :class:`~socketserver.StreamRequestHandler` classes now implements the :class:`io.BufferedIOBase` writable interface. In particular, calling :meth:`~io.BufferedIOBase.write` is now guaranteed to send the @@ -1465,7 +1465,7 @@ The new :meth:`~ssl.SSLContext.get_ciphers` method can be used to get a list of enabled ciphers in order of cipher priority. All constants and flags have been converted to :class:`~enum.IntEnum` and -:class:`~enum.IntFlags`. +:class:`~enum.IntFlag`. (Contributed by Christian Heimes in :issue:`28025`.) Server and client-side specific TLS protocols for :class:`~ssl.SSLContext` @@ -1531,8 +1531,8 @@ Stéphane Wirtel in :issue:`25485`). time ---- -The :class:`~time.struct_time` attributes :attr:`tm_gmtoff` and -:attr:`tm_zone` are now available on all platforms. +The :class:`~time.struct_time` attributes :attr:`!tm_gmtoff` and +:attr:`!tm_zone` are now available on all platforms. timeit @@ -1551,12 +1551,12 @@ between best and worst times. tkinter ------- -Added methods :meth:`~tkinter.Variable.trace_add`, -:meth:`~tkinter.Variable.trace_remove` and :meth:`~tkinter.Variable.trace_info` -in the :class:`tkinter.Variable` class. They replace old methods -:meth:`~tkinter.Variable.trace_variable`, :meth:`~tkinter.Variable.trace`, -:meth:`~tkinter.Variable.trace_vdelete` and -:meth:`~tkinter.Variable.trace_vinfo` that use obsolete Tcl commands and might +Added methods :meth:`!Variable.trace_add`, +:meth:`!Variable.trace_remove` and :meth:`!trace_info` +in the :class:`!tkinter.Variable` class. They replace old methods +:meth:`!trace_variable`, :meth:`!trace`, +:meth:`!trace_vdelete` and +:meth:`!trace_vinfo` that use obsolete Tcl commands and might not work in future versions of Tcl. (Contributed by Serhiy Storchaka in :issue:`22115`). @@ -1674,8 +1674,8 @@ urllib.request If a HTTP request has a file or iterable body (other than a bytes object) but no ``Content-Length`` header, rather than -throwing an error, :class:`~urllib.request.AbstractHTTPHandler` now -falls back to use chunked transfer encoding. +throwing an error, :class:`AbstractHTTPHandler ` +now falls back to use chunked transfer encoding. (Contributed by Demian Brecht and Rolf Krahl in :issue:`12319`.) @@ -1701,7 +1701,7 @@ warnings A new optional *source* parameter has been added to the :func:`warnings.warn_explicit` function: the destroyed object which emitted a :exc:`ResourceWarning`. A *source* attribute has also been added to -:class:`warnings.WarningMessage` (contributed by Victor Stinner in +:class:`!warnings.WarningMessage` (contributed by Victor Stinner in :issue:`26568` and :issue:`26567`). When a :exc:`ResourceWarning` warning is logged, the :mod:`tracemalloc` module is now @@ -1942,7 +1942,7 @@ Raising the :exc:`StopIteration` exception inside a generator will now generate a :exc:`DeprecationWarning`, and will trigger a :exc:`RuntimeError` in Python 3.7. See :ref:`whatsnew-pep-479` for details. -The :meth:`__aiter__` method is now expected to return an asynchronous +The :meth:`~object.__aiter__` method is now expected to return an asynchronous iterator directly instead of returning an awaitable as previously. Doing the former will trigger a :exc:`DeprecationWarning`. Backward compatibility will be removed in Python 3.7. @@ -2006,10 +2006,10 @@ deprecated. importlib ~~~~~~~~~ -The :meth:`importlib.machinery.SourceFileLoader.load_module` and -:meth:`importlib.machinery.SourcelessFileLoader.load_module` methods +The ``importlib.machinery.SourceFileLoader.load_module`` and +``importlib.machinery.SourcelessFileLoader.load_module`` methods are now deprecated. They were the only remaining implementations of -:meth:`importlib.abc.Loader.load_module` in :mod:`importlib` that had not +``importlib.abc.Loader.load_module`` in :mod:`importlib` that had not been deprecated in previous versions of Python in favour of :meth:`importlib.abc.Loader.exec_module`. @@ -2173,7 +2173,7 @@ Changes in the Python API * :c:func:`PyErr_SetImportError` now sets :exc:`TypeError` when its **msg** argument is not set. Previously only ``NULL`` was returned. -* The format of the :attr:`~codeobject.co_lnotab` attribute of code objects +* The format of the :attr:`!codeobject.co_lnotab` attribute of code objects changed to support a negative line number delta. By default, Python does not emit bytecode with a negative line number delta. Functions using :attr:`frame.f_lineno`, @@ -2189,7 +2189,7 @@ Changes in the Python API booleans being a subclass of integers, this should only be an issue if you were doing identity checks for ``1`` or ``0``. See :issue:`25768`. -* Reading the :attr:`~urllib.parse.SplitResult.port` attribute of +* Reading the :attr:`!port` attribute of :func:`urllib.parse.urlsplit` and :func:`~urllib.parse.urlparse` results now raises :exc:`ValueError` for out-of-range values, rather than returning :const:`None`. See :issue:`20059`. @@ -2197,8 +2197,8 @@ Changes in the Python API * The :mod:`!imp` module now raises a :exc:`DeprecationWarning` instead of :exc:`PendingDeprecationWarning`. -* The following modules have had missing APIs added to their :attr:`__all__` - attributes to match the documented APIs: +* The following modules have had missing APIs added to their + :attr:`~module.__all__` attributes to match the documented APIs: :mod:`calendar`, :mod:`!cgi`, :mod:`csv`, :mod:`~xml.etree.ElementTree`, :mod:`enum`, :mod:`fileinput`, :mod:`ftplib`, :mod:`logging`, :mod:`mailbox`, @@ -2253,11 +2253,13 @@ Changes in the Python API * As part of :pep:`487`, the handling of keyword arguments passed to :class:`type` (other than the metaclass hint, ``metaclass``) is now consistently delegated to :meth:`object.__init_subclass__`. This means that - :meth:`type.__new__` and :meth:`type.__init__` both now accept arbitrary - keyword arguments, but :meth:`object.__init_subclass__` (which is called from - :meth:`type.__new__`) will reject them by default. Custom metaclasses - accepting additional keyword arguments will need to adjust their calls to - :meth:`type.__new__` (whether direct or via :class:`super`) accordingly. + :meth:`type.__new__ ` and :meth:`type.__init__ + ` both now accept arbitrary keyword arguments, + but :meth:`object.__init_subclass__` (which is called from + :meth:`type.__new__ `) will reject them by default. + Custom metaclasses accepting additional keyword arguments will need to adjust + their calls to :meth:`type.__new__ ` + (whether direct or via :class:`super`) accordingly. * In ``distutils.command.sdist.sdist``, the ``default_format`` attribute has been removed and is no longer honored. Instead, the @@ -2305,7 +2307,7 @@ Changes in the Python API real-world compatibility. (Contributed by Lita Cho in :issue:`21815`.) -* The :func:`mmap.write() ` function now returns the number +* The :func:`mmap.mmap.write` function now returns the number of bytes written like other write methods. (Contributed by Jakub Stasiak in :issue:`26335`.) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index f420fa5c04479b0..3af3e6ec9cac08f 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -320,9 +320,9 @@ effort will be made to add such support. PEP 562: Customization of Access to Module Attributes ----------------------------------------------------- -Python 3.7 allows defining :meth:`__getattr__` on modules and will call +Python 3.7 allows defining :meth:`~module.__getattr__` on modules and will call it whenever a module attribute is otherwise not found. Defining -:meth:`__dir__` on modules is now also allowed. +:meth:`~module.__dir__` on modules is now also allowed. A typical example of where this may be useful is module attribute deprecation and lazy loading. @@ -409,8 +409,8 @@ PEP 560: Core Support for ``typing`` module and Generic Types Initially :pep:`484` was designed in such way that it would not introduce *any* changes to the core CPython interpreter. Now type hints and the :mod:`typing` module are extensively used by the community, so this restriction is removed. -The PEP introduces two special methods :meth:`__class_getitem__` and -``__mro_entries__``, these methods are now used by most classes and special +The PEP introduces two special methods :meth:`~object.__class_getitem__` and +:meth:`~object.__mro_entries__`, these methods are now used by most classes and special constructs in :mod:`typing`. As a result, the speed of various operations with types increased up to 7 times, the generic types can be used without metaclass conflicts, and several long standing bugs in :mod:`typing` module are @@ -571,7 +571,7 @@ decimal operations to work with the correct context in asynchronous code. dataclasses ----------- -The new :func:`~dataclasses.dataclass` decorator provides a way to declare +The new :deco:`~dataclasses.dataclass` decorator provides a way to declare *data classes*. A data class describes its attributes using class variable annotations. Its constructor and other magic methods, such as :meth:`~object.__repr__`, :meth:`~object.__eq__`, and @@ -603,8 +603,8 @@ The new :mod:`importlib.resources` module provides several new APIs and one new ABC for access to, opening, and reading *resources* inside packages. Resources are roughly similar to files inside packages, but they needn't be actual files on the physical file system. Module loaders can provide a -:meth:`get_resource_reader` function which returns -a :class:`importlib.abc.ResourceReader` instance to support this +:meth:`!get_resource_reader` function which returns +a :class:`!importlib.abc.ResourceReader` instance to support this new API. Built-in file path loaders and zip file loaders both support this. Contributed by Barry Warsaw and Brett Cannon in :issue:`32248`. @@ -830,7 +830,7 @@ The new :func:`~contextlib.nullcontext` is a simpler and faster no-op context manager than :class:`~contextlib.ExitStack`. (Contributed by Jesse-Bakker in :issue:`10049`.) -The new :func:`~contextlib.asynccontextmanager`, +The new :deco:`~contextlib.asynccontextmanager`, :class:`~contextlib.AbstractAsyncContextManager`, and :class:`~contextlib.AsyncExitStack` have been added to complement their synchronous counterparts. (Contributed @@ -841,7 +841,7 @@ and by Alexander Mohr and Ilya Kulakov in :issue:`29302`.) cProfile -------- -The :mod:`cProfile` command line now accepts ``-m module_name`` as an +The :mod:`!cProfile` command line now accepts ``-m module_name`` as an alternative to script path. (Contributed by Sanyam Khurana in :issue:`21862`.) @@ -910,9 +910,9 @@ which allows listing the names of properties which should not become enum members. (Contributed by Ethan Furman in :issue:`31801`.) -In Python 3.8, attempting to check for non-Enum objects in :class:`Enum` +In Python 3.8, attempting to check for non-Enum objects in :class:`~enum.Enum` classes will raise a :exc:`TypeError` (e.g. ``1 in Color``); similarly, -attempting to check for non-Flag objects in a :class:`Flag` member will +attempting to check for non-Flag objects in a :class:`~enum.Flag` member will raise :exc:`TypeError` (e.g. ``1 in Perm.RW``); currently, both operations return :const:`False` instead and are deprecated. (Contributed by Ethan Furman in :issue:`33217`.) @@ -921,7 +921,7 @@ return :const:`False` instead and are deprecated. functools --------- -:func:`functools.singledispatch` now supports registering implementations +:deco:`functools.singledispatch` now supports registering implementations using type annotations. (Contributed by Łukasz Langa in :issue:`32227`.) @@ -969,7 +969,7 @@ uses the current working directory. (Contributed by Stéphane Wirtel and Julien Palard in :issue:`28707`.) The new :class:`ThreadingHTTPServer ` class -uses threads to handle requests using :class:`~socketserver.ThreadingMixin`. +uses threads to handle requests using :class:`~socketserver.ThreadingMixIn`. It is used when ``http.server`` is run with ``-m``. (Contributed by Julien Palard in :issue:`31639`.) @@ -1043,7 +1043,7 @@ window are shown and hidden in the Options menu. importlib --------- -The :class:`importlib.abc.ResourceReader` ABC was introduced to +The :class:`!importlib.abc.ResourceReader` ABC was introduced to support the loading of resources from packages. See also :ref:`whatsnew37_importlib_resources`. (Contributed by Barry Warsaw, Brett Cannon in :issue:`32248`.) @@ -1052,12 +1052,12 @@ support the loading of resources from packages. See also lacks a spec. (Contributed by Garvit Khatri in :issue:`29851`.) -:func:`importlib.find_spec` now raises :exc:`ModuleNotFoundError` instead of +:func:`importlib.util.find_spec` now raises :exc:`ModuleNotFoundError` instead of :exc:`AttributeError` if the specified parent module is not a package (i.e. lacks a ``__path__`` attribute). (Contributed by Milan Oberkirch in :issue:`30436`.) -The new :func:`importlib.source_hash` can be used to compute the hash of +The new :func:`importlib.util.source_hash` can be used to compute the hash of the passed source. A :ref:`hash-based .pyc file ` embeds the value returned by this function. @@ -1148,7 +1148,7 @@ running. (Contributed by Antoine Pitrou in :issue:`30596`.) The new :meth:`Process.kill() ` method can -be used to terminate the process using the :data:`SIGKILL` signal on Unix. +be used to terminate the process using the :data:`~signal.SIGKILL` signal on Unix. (Contributed by Vitor Pereira in :issue:`30794`.) Non-daemonic threads created by :class:`~multiprocessing.Process` are now @@ -1280,9 +1280,10 @@ This function should be used instead of :func:`os.close` for better compatibility across platforms. (Contributed by Christian Heimes in :issue:`32454`.) -The :mod:`socket` module now exposes the :const:`socket.TCP_CONGESTION` -(Linux 2.6.13), :const:`socket.TCP_USER_TIMEOUT` (Linux 2.6.37), and -:const:`socket.TCP_NOTSENT_LOWAT` (Linux 3.12) constants. +The :mod:`socket` module now exposes the :ref:`socket.TCP_CONGESTION +` (Linux 2.6.13), :ref:`socket.TCP_USER_TIMEOUT +` (Linux 2.6.37), and :ref:`socket.TCP_NOTSENT_LOWAT +` (Linux 3.12) constants. (Contributed by Omar Sandoval in :issue:`26273` and Nathaniel J. Smith in :issue:`29728`.) @@ -1298,11 +1299,14 @@ by default. socketserver ------------ -:meth:`socketserver.ThreadingMixIn.server_close` now waits until all non-daemon -threads complete. :meth:`socketserver.ForkingMixIn.server_close` now waits +:meth:`socketserver.ThreadingMixIn.server_close +` now waits until all non-daemon +threads complete. :meth:`socketserver.ForkingMixIn.server_close +` now waits until all child processes complete. -Add a new :attr:`socketserver.ForkingMixIn.block_on_close` class attribute to +Add a new :attr:`socketserver.ForkingMixIn.block_on_close +` class attribute to :class:`socketserver.ForkingMixIn` and :class:`socketserver.ThreadingMixIn` classes. Set the class attribute to ``False`` to get the pre-3.7 behaviour. @@ -1323,7 +1327,7 @@ ssl --- The :mod:`ssl` module now uses OpenSSL's builtin API instead of -:func:`~ssl.match_hostname` to check a host name or an IP address. Values +:func:`!match_hostname` to check a host name or an IP address. Values are validated during TLS handshake. Any certificate validation error including failing the host name check now raises :exc:`~ssl.SSLCertVerificationError` and aborts the handshake with a proper @@ -1341,7 +1345,7 @@ Host name validation can be customized with The ``ssl`` module no longer sends IP addresses in SNI TLS extension. (Contributed by Christian Heimes in :issue:`32185`.) -:func:`~ssl.match_hostname` no longer supports partial wildcards like +:func:`!match_hostname` no longer supports partial wildcards like ``www*.example.org``. (Contributed by Mandeep Singh in :issue:`23033` and Christian Heimes in :issue:`31399`.) @@ -1438,7 +1442,7 @@ The new :func:`sys.get_coroutine_origin_tracking_depth` function returns the current coroutine origin tracking depth, as set by the new :func:`sys.set_coroutine_origin_tracking_depth`. :mod:`asyncio` has been converted to use this new API instead of -the deprecated :func:`sys.set_coroutine_wrapper`. +the deprecated :func:`!sys.set_coroutine_wrapper`. (Contributed by Nathaniel J. Smith in :issue:`32591`.) @@ -1615,7 +1619,7 @@ external entities by default. xml.etree --------- -:ref:`ElementPath ` predicates in the :meth:`find` +:ref:`ElementPath ` predicates in the :meth:`!find` methods can now compare text of the current node with ``[. = "text"]``, not only text in children. Predicates also allow adding spaces for better readability. (Contributed by Stefan Behnel in :issue:`31648`.) @@ -1624,7 +1628,7 @@ better readability. (Contributed by Stefan Behnel in :issue:`31648`.) xmlrpc.server ------------- -:meth:`SimpleXMLRPCDispatcher.register_function ` +:meth:`!SimpleXMLRPCDispatcher.register_function` can now be used as a decorator. (Contributed by Xiang Zhang in :issue:`7769`.) @@ -1682,15 +1686,15 @@ The :mod:`tracemalloc` now exposes a C API through the new functions. (Contributed by Victor Stinner in :issue:`30054`.) -The new :c:func:`import__find__load__start` and -:c:func:`import__find__load__done` static markers can be used to trace -module imports. +The new :ref:`import__find__load__start ` and +:ref:`import__find__load__done ` static markers can be used +to trace module imports. (Contributed by Christian Heimes in :issue:`31574`.) The fields :c:member:`!name` and :c:member:`!doc` of structures :c:type:`PyMemberDef`, :c:type:`PyGetSetDef`, :c:type:`PyStructSequence_Field`, :c:type:`PyStructSequence_Desc`, -and :c:struct:`wrapperbase` are now of type ``const char *`` rather of +and :c:struct:`!wrapperbase` are now of type ``const char *`` rather of ``char *``. (Contributed by Serhiy Storchaka in :issue:`28761`.) The result of :c:func:`PyUnicode_AsUTF8AndSize` and :c:func:`PyUnicode_AsUTF8` @@ -1719,8 +1723,8 @@ Added C API support for timezones with timezone constructors and access to the UTC singleton with :c:data:`PyDateTime_TimeZone_UTC`. Contributed by Paul Ganssle in :issue:`10381`. -The type of results of :c:func:`PyThread_start_new_thread` and -:c:func:`PyThread_get_thread_ident`, and the *id* parameter of +The type of results of :c:func:`!PyThread_start_new_thread` and +:c:func:`!PyThread_get_thread_ident`, and the *id* parameter of :c:func:`PyThreadState_SetAsyncExc` changed from :c:expr:`long` to :c:expr:`unsigned long`. (Contributed by Serhiy Storchaka in :issue:`6532`.) @@ -1847,8 +1851,8 @@ make the creation of named tuples 4 to 6 times faster. (Contributed by Jelle Zijlstra with further improvements by INADA Naoki, Serhiy Storchaka, and Raymond Hettinger in :issue:`28638`.) -:meth:`date.fromordinal` and :meth:`date.fromtimestamp` are now up to -30% faster in the common case. +:meth:`datetime.date.fromordinal` and :meth:`datetime.date.fromtimestamp` +are now up to 30% faster in the common case. (Contributed by Paul Ganssle in :issue:`32403`.) The :func:`os.fwalk` function is now up to 2 times faster thanks to @@ -1997,9 +2001,9 @@ modes (this will be an error in future Python releases). enum ---- -In Python 3.8, attempting to check for non-Enum objects in :class:`Enum` +In Python 3.8, attempting to check for non-Enum objects in :class:`~enum.Enum` classes will raise a :exc:`TypeError` (e.g. ``1 in Color``); similarly, -attempting to check for non-Flag objects in a :class:`Flag` member will +attempting to check for non-Flag objects in a :class:`~enum.Flag` member will raise :exc:`TypeError` (e.g. ``1 in Perm.RW``); currently, both operations return :const:`False` instead. (Contributed by Ethan Furman in :issue:`33217`.) @@ -2028,20 +2032,20 @@ both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. (Contributed by Matthias Bussonnier in :issue:`29576`.) The :class:`importlib.abc.ResourceLoader` ABC has been deprecated in -favour of :class:`importlib.abc.ResourceReader`. +favour of :class:`!importlib.abc.ResourceReader`. locale ------ -:func:`locale.format` has been deprecated, use :meth:`locale.format_string` +:func:`!locale.format` has been deprecated, use :meth:`locale.format_string` instead. (Contributed by Garvit in :issue:`10379`.) macpath ------- -The :mod:`macpath` is now deprecated and will be removed in Python 3.8. +The :mod:`!macpath` is now deprecated and will be removed in Python 3.8. (Contributed by Chi Hsuan Yen in :issue:`9850`.) @@ -2066,7 +2070,7 @@ if the passed argument is larger than 16 bits, an exception will be raised. ssl --- -:func:`ssl.wrap_socket` is deprecated. Use +:func:`!ssl.wrap_socket` is deprecated. Use :meth:`ssl.SSLContext.wrap_socket` instead. (Contributed by Christian Heimes in :issue:`28124`.) @@ -2082,8 +2086,8 @@ Use :func:`!sunau.open` instead. sys --- -Deprecated :func:`sys.set_coroutine_wrapper` and -:func:`sys.get_coroutine_wrapper`. +Deprecated :func:`!sys.set_coroutine_wrapper` and +:func:`!sys.get_coroutine_wrapper`. The undocumented ``sys.callstats()`` function has been deprecated and will be removed in a future Python version. @@ -2093,7 +2097,7 @@ will be removed in a future Python version. wave ---- -:func:`wave.openfp` has been deprecated and will be removed in Python 3.9. +:func:`!wave.openfp` has been deprecated and will be removed in Python 3.9. Use :func:`wave.open` instead. (Contributed by Brian Curtin in :issue:`31985`.) @@ -2173,8 +2177,8 @@ The following features and APIs have been removed from Python 3.7: * Removed previously deprecated in Python 2.4 classes ``Plist``, ``Dict`` and ``_InternalDict`` in the :mod:`plistlib` module. Dict values in the result - of functions :func:`~plistlib.readPlist` and - :func:`~plistlib.readPlistFromBytes` are now normal dicts. You no longer + of functions :func:`!readPlist` and + :func:`!readPlistFromBytes` are now normal dicts. You no longer can use attribute access to access items of these dictionaries. * The ``asyncio.windows_utils.socketpair()`` function has been @@ -2191,7 +2195,7 @@ The following features and APIs have been removed from Python 3.7: * Direct instantiation of :class:`ssl.SSLSocket` and :class:`ssl.SSLObject` objects is now prohibited. The constructors were never documented, tested, or designed as public constructors. Users were supposed to use - :func:`ssl.wrap_socket` or :class:`ssl.SSLContext`. + :func:`!ssl.wrap_socket` or :class:`ssl.SSLContext`. (Contributed by Christian Heimes in :issue:`32951`.) * The unused ``distutils`` ``install_misc`` command has been removed. @@ -2275,15 +2279,18 @@ Changes in Python Behavior Changes in the Python API ------------------------- -* :meth:`socketserver.ThreadingMixIn.server_close` now waits until all +* :meth:`socketserver.ThreadingMixIn.server_close + ` now waits until all non-daemon threads complete. Set the new :attr:`socketserver.ThreadingMixIn.block_on_close` class attribute to ``False`` to get the pre-3.7 behaviour. (Contributed by Victor Stinner in :issue:`31233` and :issue:`33540`.) -* :meth:`socketserver.ForkingMixIn.server_close` now waits until all +* :meth:`socketserver.ForkingMixIn.server_close + ` now waits until all child processes complete. Set the new - :attr:`socketserver.ForkingMixIn.block_on_close` class attribute to ``False`` + :attr:`socketserver.ForkingMixIn.block_on_close + ` class attribute to ``False`` to get the pre-3.7 behaviour. (Contributed by Victor Stinner in :issue:`31151` and :issue:`33540`.) @@ -2476,7 +2483,7 @@ avoiding possible problems use new functions :c:func:`PySlice_Unpack` and CPython bytecode changes ------------------------ -There are two new opcodes: :opcode:`LOAD_METHOD` and :opcode:`!CALL_METHOD`. +There are two new opcodes: :opcode:`!LOAD_METHOD` and :opcode:`!CALL_METHOD`. (Contributed by Yury Selivanov and INADA Naoki in :issue:`26110`.) The :opcode:`!STORE_ANNOTATION` opcode has been removed. diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index bc2eb1d0e263f06..42d98ff94ec0d1b 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -50,7 +50,6 @@ For full details, see the :ref:`changelog `. .. testsetup:: - from datetime import date from math import cos, radians from unicodedata import normalize import re @@ -208,14 +207,15 @@ subdirectories). Debug build uses the same ABI as release build ----------------------------------------------- -Python now uses the same ABI whether it's built in release or debug mode. On -Unix, when Python is built in debug mode, it is now possible to load C -extensions built in release mode and C extensions built using the stable ABI. +The ABI of Python :ref:`debug builds ` is now compatible with +Python release builds. On Unix, when Python is built in debug mode, it is now +possible to load C extensions built in release mode and C extensions built +using the stable ABI. The inverse is not true, as debug builds expose +additional symbols not available in release builds. -Release builds and :ref:`debug builds ` are now ABI compatible: defining the -``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro, which -introduces the only ABI incompatibility. The ``Py_TRACE_REFS`` macro, which -adds the :func:`sys.getobjects` function and the :envvar:`PYTHONDUMPREFS` +Defining the ``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro, +which introduces the only ABI incompatibility. The ``Py_TRACE_REFS`` macro, +which adds the :func:`sys.getobjects` function and the :envvar:`PYTHONDUMPREFS` environment variable, can be set using the new :option:`./configure --with-trace-refs <--with-trace-refs>` build option. (Contributed by Victor Stinner in :issue:`36465`.) @@ -259,15 +259,16 @@ Added an ``=`` specifier to :term:`f-string`\s. An f-string such as ``f'{expr=}'`` will expand to the text of the expression, an equal sign, then the representation of the evaluated expression. For example: + >>> import datetime as dt >>> user = 'eric_idle' - >>> member_since = date(1975, 7, 31) + >>> member_since = dt.date(1975, 7, 31) >>> f'{user=} {member_since=}' "user='eric_idle' member_since=datetime.date(1975, 7, 31)" The usual :ref:`f-string format specifiers ` allow more control over how the result of the expression is displayed:: - >>> delta = date.today() - member_since + >>> delta = dt.date.today() - member_since >>> f'{user=!s} {delta.days=:,d}' 'user=eric_idle delta.days=16,075' @@ -536,7 +537,7 @@ Other Language Changes element is a callable with a ``(obj, state)`` signature. This allows the direct control over the state-updating behavior of a specific object. If not *None*, this callable will have priority over the object's - :meth:`~__setstate__` method. + :meth:`~object.__setstate__` method. (Contributed by Pierre Glaser and Olivier Grisel in :issue:`35900`.) New Modules @@ -763,7 +764,7 @@ these are the inverse of each class's ``isocalendar`` method. functools --------- -:func:`functools.lru_cache` can now be used as a straight decorator rather +:deco:`functools.lru_cache` can now be used as a straight decorator rather than as a function returning a decorator. So both of these are now supported:: @lru_cache @@ -776,7 +777,7 @@ than as a function returning a decorator. So both of these are now supported:: (Contributed by Raymond Hettinger in :issue:`36772`.) -Added a new :func:`functools.cached_property` decorator, for computed properties +Added a new :deco:`functools.cached_property` decorator, for computed properties cached for the life of the instance. :: import functools @@ -892,7 +893,7 @@ inspect The :func:`inspect.getdoc` function can now find docstrings for ``__slots__`` if that attribute is a :class:`dict` where the values are docstrings. This provides documentation options similar to what we already have -for :func:`property`, :func:`classmethod`, and :func:`staticmethod`:: +for :deco:`property`, :deco:`classmethod`, and :deco:`staticmethod`:: class AudioClip: __slots__ = {'bit_rate': 'expressed in kilohertz to one decimal place', @@ -1035,9 +1036,9 @@ symlinks and directory junctions) has been delegated to the operating system. Specifically, :func:`os.stat` will now traverse anything supported by the operating system, while :func:`os.lstat` will only open reparse points that identify as "name surrogates" while others are opened as for :func:`os.stat`. -In all cases, :attr:`stat_result.st_mode` will only have ``S_IFLNK`` set for +In all cases, :attr:`os.stat_result.st_mode` will only have ``S_IFLNK`` set for symbolic links and not other kinds of reparse points. To identify other kinds -of reparse point, check the new :attr:`stat_result.st_reparse_tag` attribute. +of reparse point, check the new :attr:`os.stat_result.st_reparse_tag` attribute. On Windows, :func:`os.readlink` is now able to read directory junctions. Note that :func:`~os.path.islink` will return ``False`` for directory junctions, @@ -1285,20 +1286,20 @@ now matches what the C tokenizer does internally. tkinter ------- -Added methods :meth:`~tkinter.Spinbox.selection_from`, -:meth:`~tkinter.Spinbox.selection_present`, -:meth:`~tkinter.Spinbox.selection_range` and -:meth:`~tkinter.Spinbox.selection_to` -in the :class:`tkinter.Spinbox` class. +Added methods :meth:`!selection_from`, +:meth:`!selection_present`, +:meth:`!selection_range` and +:meth:`!selection_to` +in the :class:`!tkinter.Spinbox` class. (Contributed by Juliette Monsel in :issue:`34829`.) -Added method :meth:`~tkinter.Canvas.moveto` -in the :class:`tkinter.Canvas` class. +Added method :meth:`!moveto` +in the :class:`!tkinter.Canvas` class. (Contributed by Juliette Monsel in :issue:`23831`.) -The :class:`tkinter.PhotoImage` class now has -:meth:`~tkinter.PhotoImage.transparency_get` and -:meth:`~tkinter.PhotoImage.transparency_set` methods. (Contributed by +The :class:`!tkinter.PhotoImage` class now has +:meth:`!transparency_get` and +:meth:`!transparency_set` methods. (Contributed by Zackery Spytz in :issue:`25451`.) @@ -1332,14 +1333,14 @@ The :mod:`typing` module incorporates several new features: ... * "Final" variables, functions, methods and classes. See :pep:`591`, - :class:`typing.Final` and :func:`typing.final`. + :class:`typing.Final` and :deco:`typing.final`. The final qualifier instructs a static type checker to restrict subclassing, overriding, or reassignment:: pi: Final[float] = 3.1415926536 * Protocol definitions. See :pep:`544`, :class:`typing.Protocol` and - :func:`typing.runtime_checkable`. Simple ABCs like + :deco:`typing.runtime_checkable`. Simple ABCs like :class:`typing.SupportsInt` are now ``Protocol`` subclasses. * New protocol class :class:`typing.SupportsIndex`. @@ -1432,7 +1433,7 @@ and ``{namespace}*`` which returns all tags in the given namespace. (Contributed by Stefan Behnel in :issue:`28238`.) The :mod:`xml.etree.ElementTree` module provides a new function -:func:`–xml.etree.ElementTree.canonicalize` that implements C14N 2.0. +:func:`~xml.etree.ElementTree.canonicalize` that implements C14N 2.0. (Contributed by Stefan Behnel in :issue:`13611`.) The target object of :class:`xml.etree.ElementTree.XMLParser` can @@ -1573,7 +1574,7 @@ Build and C API Changes * :c:func:`Py_INCREF`, :c:func:`Py_DECREF` * :c:func:`Py_XINCREF`, :c:func:`Py_XDECREF` - * :c:func:`PyObject_INIT`, :c:func:`PyObject_INIT_VAR` + * :c:macro:`!PyObject_INIT`, :c:macro:`!PyObject_INIT_VAR` * Private functions: :c:func:`!_PyObject_GC_TRACK`, :c:func:`!_PyObject_GC_UNTRACK`, :c:func:`!_Py_Dealloc` @@ -1677,7 +1678,7 @@ Deprecated constant nodes. (Contributed by Serhiy Storchaka in :issue:`36917`.) -* The :func:`asyncio.coroutine` :term:`decorator` is deprecated and will be +* The :deco:`!asyncio.coroutine` :term:`decorator` is deprecated and will be removed in version 3.10. Instead of ``@asyncio.coroutine``, use :keyword:`async def` instead. (Contributed by Andrew Svetlov in :issue:`36921`.) @@ -1697,22 +1698,22 @@ Deprecated (Contributed by Yury Selivanov in :issue:`34790`.) * The following functions and methods are deprecated in the :mod:`gettext` - module: :func:`~gettext.lgettext`, :func:`~gettext.ldgettext`, - :func:`~gettext.lngettext` and :func:`~gettext.ldngettext`. + module: :func:`!lgettext`, :func:`!ldgettext`, + :func:`!lngettext` and :func:`!ldngettext`. They return encoded bytes, and it's possible that you will get unexpected Unicode-related exceptions if there are encoding problems with the translated strings. It's much better to use alternatives which return Unicode strings in Python 3. These functions have been broken for a long time. - Function :func:`~gettext.bind_textdomain_codeset`, methods - :meth:`~gettext.NullTranslations.output_charset` and - :meth:`~gettext.NullTranslations.set_output_charset`, and the *codeset* + Function :func:`!bind_textdomain_codeset`, methods + :meth:`!NullTranslations.output_charset` and + :meth:`!NullTranslations.set_output_charset`, and the *codeset* parameter of functions :func:`~gettext.translation` and :func:`~gettext.install` are also deprecated, since they are only used for the ``l*gettext()`` functions. (Contributed by Serhiy Storchaka in :issue:`33710`.) -* The :meth:`~threading.Thread.isAlive` method of :class:`threading.Thread` +* The :meth:`!isAlive` method of :class:`threading.Thread` has been deprecated. (Contributed by Donghee Na in :issue:`35283`.) @@ -1727,7 +1728,7 @@ Deprecated * Deprecated passing the following arguments as keyword arguments: - *func* in :func:`functools.partialmethod`, :func:`weakref.finalize`, - :meth:`profile.Profile.runcall`, :meth:`cProfile.Profile.runcall`, + :meth:`profile.Profile.runcall`, :meth:`!cProfile.Profile.runcall`, :meth:`bdb.Bdb.runcall`, :meth:`trace.Trace.runfunc` and :func:`curses.wrapper`. - *function* in :meth:`unittest.TestCase.addCleanup`. @@ -1735,11 +1736,11 @@ Deprecated :class:`concurrent.futures.ThreadPoolExecutor` and :class:`concurrent.futures.ProcessPoolExecutor`. - *callback* in :meth:`contextlib.ExitStack.callback`, - :meth:`contextlib.AsyncExitStack.callback` and + :meth:`!contextlib.AsyncExitStack.callback` and :meth:`contextlib.AsyncExitStack.push_async_callback`. - - *c* and *typeid* in the :meth:`~multiprocessing.managers.Server.create` - method of :class:`multiprocessing.managers.Server` and - :class:`multiprocessing.managers.SharedMemoryServer`. + - *c* and *typeid* in the :meth:`!create` + method of :class:`!multiprocessing.managers.Server` and + :class:`!multiprocessing.managers.SharedMemoryServer`. - *obj* in :func:`weakref.finalize`. In future releases of Python, they will be :ref:`positional-only @@ -1757,14 +1758,14 @@ The following features and APIs have been removed from Python 3.8: able to import from collections was marked for removal in 3.8, but has been delayed to 3.9. (See :gh:`81134`.) -* The :mod:`macpath` module, deprecated in Python 3.7, has been removed. +* The :mod:`!macpath` module, deprecated in Python 3.7, has been removed. (Contributed by Victor Stinner in :issue:`35471`.) -* The function :func:`platform.popen` has been removed, after having been +* The function :func:`!platform.popen` has been removed, after having been deprecated since Python 3.3: use :func:`os.popen` instead. (Contributed by Victor Stinner in :issue:`35345`.) -* The function :func:`time.clock` has been removed, after having been +* The function :func:`!time.clock` has been removed, after having been deprecated since Python 3.3: use :func:`time.perf_counter` or :func:`time.process_time` instead, depending on your requirements, to have well-defined behavior. @@ -1800,8 +1801,8 @@ The following features and APIs have been removed from Python 3.8: :func:`fileinput.FileInput` which was ignored and deprecated since Python 3.6 has been removed. :issue:`36952` (Contributed by Matthias Bussonnier.) -* The functions :func:`sys.set_coroutine_wrapper` and - :func:`sys.get_coroutine_wrapper` deprecated in Python 3.7 have been removed; +* The functions :func:`!sys.set_coroutine_wrapper` and + :func:`!sys.get_coroutine_wrapper` deprecated in Python 3.7 have been removed; :issue:`36933` (Contributed by Matthias Bussonnier.) @@ -1864,9 +1865,10 @@ Changes in the Python API * :class:`subprocess.Popen` can now use :func:`os.posix_spawn` in some cases for better performance. On Windows Subsystem for Linux and QEMU User - Emulation, the :class:`Popen` constructor using :func:`os.posix_spawn` no longer raises an - exception on errors like "missing program". Instead the child process fails with a - non-zero :attr:`~Popen.returncode`. + Emulation, the :class:`~subprocess.Popen` constructor using + :func:`os.posix_spawn` no longer raises an exception on errors like + "missing program". Instead the child process fails with a + non-zero :attr:`~subprocess.Popen.returncode`. (Contributed by Joannah Nanjekye and Victor Stinner in :issue:`35537`.) * The *preexec_fn* argument of * :class:`subprocess.Popen` is no longer @@ -1875,11 +1877,11 @@ Changes in the Python API (Contributed by Eric Snow in :issue:`34651`, modified by Christian Heimes in :issue:`37951`.) -* The :meth:`imap.IMAP4.logout` method no longer silently ignores arbitrary +* The :meth:`imaplib.IMAP4.logout` method no longer silently ignores arbitrary exceptions. (Contributed by Victor Stinner in :issue:`36348`.) -* The function :func:`platform.popen` has been removed, after having been deprecated since +* The function :func:`!platform.popen` has been removed, after having been deprecated since Python 3.3: use :func:`os.popen` instead. (Contributed by Victor Stinner in :issue:`35345`.) @@ -1894,9 +1896,11 @@ Changes in the Python API specialized methods like :meth:`~tkinter.ttk.Treeview.selection_set` for changing the selection. (Contributed by Serhiy Storchaka in :issue:`31508`.) -* The :meth:`writexml`, :meth:`toxml` and :meth:`toprettyxml` methods of - :mod:`xml.dom.minidom`, and the :meth:`write` method of :mod:`xml.etree`, - now preserve the attribute order specified by the user. +* The :meth:`~xml.dom.minidom.Node.writexml`, :meth:`~xml.dom.minidom.Node.toxml` + and :meth:`~xml.dom.minidom.Node.toprettyxml` methods of + :mod:`xml.dom.minidom` and the :meth:`~xml.etree.ElementTree.ElementTree.write` + method of :mod:`xml.etree.ElementTree` now preserve the attribute order + specified by the user. (Contributed by Diego Rojas and Raymond Hettinger in :issue:`34160`.) * A :mod:`dbm.dumb` database opened with flags ``'r'`` is now read-only. @@ -1916,8 +1920,8 @@ Changes in the Python API ``type.__new__``. A :exc:`DeprecationWarning` was emitted in Python 3.6--3.7. (Contributed by Serhiy Storchaka in :issue:`23722`.) -* The :class:`cProfile.Profile` class can now be used as a context - manager. (Contributed by Scott Sanderson in :issue:`29235`.) +* The :class:`cProfile.Profile ` class can now be used as + a context manager. (Contributed by Scott Sanderson in :issue:`29235`.) * :func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2`, :func:`shutil.copytree` and :func:`shutil.move` use platform-specific @@ -1952,7 +1956,7 @@ Changes in the Python API (Contributed by Christian Heimes in :issue:`17239`.) * Deleting a key from a read-only :mod:`dbm` database (:mod:`dbm.dumb`, - :mod:`dbm.gnu` or :mod:`dbm.ndbm`) raises :attr:`error` (:exc:`dbm.dumb.error`, + :mod:`dbm.gnu` or :mod:`dbm.ndbm`) raises :attr:`!error` (:exc:`dbm.dumb.error`, :exc:`dbm.gnu.error` or :exc:`dbm.ndbm.error`) instead of :exc:`KeyError`. (Contributed by Xiang Zhang in :issue:`33106`.) @@ -2044,7 +2048,7 @@ Changes in the C API :c:func:`PyType_FromSpec`) hold a reference to their type object. Increasing the reference count of these type objects has been moved from :c:func:`PyType_GenericAlloc` to the more low-level functions, - :c:func:`PyObject_Init` and :c:func:`PyObject_INIT`. + :c:func:`PyObject_Init` and :c:macro:`!PyObject_INIT`. This makes types created through :c:func:`PyType_FromSpec` behave like other classes in managed code. @@ -2064,7 +2068,7 @@ Changes in the C API This may happen after calling :c:macro:`PyObject_New`, :c:macro:`PyObject_NewVar`, :c:func:`PyObject_GC_New`, :c:func:`PyObject_GC_NewVar`, or any other custom allocator that uses - :c:func:`PyObject_Init` or :c:func:`PyObject_INIT`. + :c:func:`PyObject_Init` or :c:macro:`!PyObject_INIT`. Example: diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 40d4a27bff9fee5..49a52b7504bc951 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -282,20 +282,20 @@ the standard library. It adds :class:`zoneinfo.ZoneInfo`, a concrete Example:: >>> from zoneinfo import ZoneInfo - >>> from datetime import datetime, timedelta + >>> import datetime as dt >>> # Daylight saving time - >>> dt = datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) - >>> print(dt) + >>> when = dt.datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) + >>> print(when) 2020-10-31 12:00:00-07:00 - >>> dt.tzname() + >>> when.tzname() 'PDT' >>> # Standard time - >>> dt += timedelta(days=7) - >>> print(dt) + >>> when += dt.timedelta(days=7) + >>> print(when) 2020-11-07 12:00:00-08:00 - >>> print(dt.tzname()) + >>> print(when.tzname()) PST diff --git a/Doc/whatsnew/index.rst b/Doc/whatsnew/index.rst index 38194db670b8390..420876a9b3ecd00 100644 --- a/Doc/whatsnew/index.rst +++ b/Doc/whatsnew/index.rst @@ -11,6 +11,7 @@ anyone wishing to stay up-to-date after a new release. .. toctree:: :maxdepth: 2 + 3.16.rst 3.15.rst 3.14.rst 3.13.rst diff --git a/Grammar/python.gram b/Grammar/python.gram index ff54e42111005ab..9bf3a67939fcf37 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -94,10 +94,12 @@ func_type[mod_ty]: '(' a=[type_expressions] ')' '->' b=expression NEWLINE* ENDMA # GENERAL STATEMENTS # ================== -statements[asdl_stmt_seq*]: a=statement+ { _PyPegen_register_stmts(p, (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a)) } +statements[asdl_stmt_seq*]: a=statement+ { (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a) } statement[asdl_stmt_seq*]: - | a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } + | a=compound_stmt { _PyPegen_register_stmts(p , + (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) + ) } | a[asdl_stmt_seq*]=simple_stmts { a } single_compound_stmt[asdl_stmt_seq*]: @@ -119,9 +121,9 @@ simple_stmts[asdl_stmt_seq*]: simple_stmt[stmt_ty] (memo): | assignment | &"type" type_alias + | &('import' | 'from' | "lazy") import_stmt | e=star_expressions { _PyAST_Expr(e, EXTRA) } | &'return' return_stmt - | &('import' | 'from') import_stmt | &'raise' raise_stmt | &'pass' pass_stmt | &'del' del_stmt @@ -210,9 +212,11 @@ del_stmt[stmt_ty]: yield_stmt[stmt_ty]: y=yield_expr { _PyAST_Expr(y, EXTRA) } -assert_stmt[stmt_ty]: 'assert' a=expression b=[',' z=expression { z }] { _PyAST_Assert(a, b, EXTRA) } +assert_stmt[stmt_ty]: + | invalid_assert_stmt + | 'assert' a=expression b=[',' z=expression { z }] { _PyAST_Assert(a, b, EXTRA) } -import_stmt[stmt_ty]: +import_stmt[stmt_ty](memo): | invalid_import | import_name | import_from @@ -220,13 +224,15 @@ import_stmt[stmt_ty]: # Import statements # ----------------- -import_name[stmt_ty]: 'import' a=dotted_as_names { _PyAST_Import(a, EXTRA) } +import_name[stmt_ty]: + | lazy="lazy"? 'import' a=dotted_as_names { _PyAST_Import(a, lazy ? 1 : 0, EXTRA) } + # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS import_from[stmt_ty]: - | 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets { - _PyPegen_checked_future_import(p, b->v.Name.id, c, _PyPegen_seq_count_dots(a), EXTRA) } - | 'from' a=('.' | '...')+ 'import' b=import_from_targets { - _PyAST_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), EXTRA) } + | lazy="lazy"? 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets { + _PyPegen_checked_future_import(p, b->v.Name.id, c, _PyPegen_seq_count_dots(a), lazy, EXTRA) } + | lazy="lazy"? 'from' a=('.' | '...')+ 'import' b=import_from_targets { + _PyAST_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), lazy ? 1 : 0, EXTRA) } import_from_targets[asdl_alias_seq*]: | '(' a=import_from_as_names [','] ')' { a } | import_from_as_names !',' @@ -548,10 +554,12 @@ complex_number[expr_ty]: signed_number[expr_ty]: | NUMBER + | '+' number=NUMBER { number } | '-' number=NUMBER { _PyAST_UnaryOp(USub, number, EXTRA) } signed_real_number[expr_ty]: | real_number + | '+' real=real_number { real } | '-' real=real_number { _PyAST_UnaryOp(USub, real, EXTRA) } real_number[expr_ty]: @@ -559,6 +567,7 @@ real_number[expr_ty]: imaginary_number[expr_ty]: | imag=NUMBER { _PyPegen_ensure_imaginary(p, imag) } + | '+' imag=NUMBER { _PyPegen_ensure_imaginary(p, imag) } capture_pattern[pattern_ty]: | target=pattern_capture_target { _PyAST_MatchAs(NULL, target->v.Name.id, EXTRA) } @@ -622,6 +631,7 @@ mapping_pattern[pattern_ty]: CHECK(asdl_pattern_seq*, _PyPegen_get_patterns(p, items)), NULL, EXTRA) } + | invalid_mapping_pattern items_pattern[asdl_seq*]: | ','.key_value_pattern+ @@ -704,12 +714,16 @@ expressions[expr_ty]: | expression expression[expr_ty] (memo): + | invalid_if_expression | invalid_expression | invalid_legacy_expression - | a=disjunction 'if' b=disjunction 'else' c=expression { _PyAST_IfExp(b, a, c, EXTRA) } + | if_expression | disjunction | lambdef +if_expression[expr_ty]: + | a=disjunction 'if' b=disjunction 'else' c=expression { _PyAST_IfExp(b, a, c, EXTRA) } + yield_expr[expr_ty]: | 'yield' 'from' a=expression { _PyAST_YieldFrom(a, EXTRA) } | 'yield' a=[star_expressions] { _PyAST_Yield(a, EXTRA) } @@ -726,10 +740,16 @@ star_expression[expr_ty] (memo): star_named_expressions[asdl_expr_seq*]: a[asdl_expr_seq*]=','.star_named_expression+ [','] { a } +star_named_expressions_sequence[asdl_expr_seq*]: a[asdl_expr_seq*]=','.star_named_expression_sequence+ [','] { a } + star_named_expression[expr_ty]: | '*' a=bitwise_or { _PyAST_Starred(a, Load, EXTRA) } | named_expression +star_named_expression_sequence[expr_ty]: + | invalid_starred_expression_unpacking_sequence + | star_named_expression + assignment_expression[expr_ty]: | a=NAME ':=' ~ b=expression { CHECK_VERSION(expr_ty, 8, "Assignment expressions are", @@ -877,9 +897,9 @@ atom[expr_ty]: | 'None' { _PyAST_Constant(Py_None, NULL, EXTRA) } | &(STRING|FSTRING_START|TSTRING_START) strings | NUMBER - | &'(' (tuple | group | genexp) - | &'[' (list | listcomp) - | &'{' (dict | set | dictcomp | setcomp) + | &'(' (genexp | tuple | group) + | &'[' (listcomp | list) + | &'{' (dictcomp | setcomp | dict | set) | '...' { _PyAST_Constant(Py_Ellipsis, NULL, EXTRA) } group[expr_ty]: @@ -993,13 +1013,13 @@ strings[expr_ty] (memo): | a[asdl_expr_seq*]=tstring+ { _PyPegen_concatenate_tstrings(p, a, EXTRA) } list[expr_ty]: - | '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) } + | '[' a=[star_named_expressions_sequence] ']' { _PyAST_List(a, Load, EXTRA) } tuple[expr_ty]: - | '(' a=[y=star_named_expression ',' z=[star_named_expressions] { _PyPegen_seq_insert_in_front(p, y, z) } ] ')' { + | '(' a=[y=star_named_expression_sequence ',' z=[star_named_expressions_sequence] { _PyPegen_seq_insert_in_front(p, y, z) } ] ')' { _PyAST_Tuple(a, Load, EXTRA) } -set[expr_ty]: '{' a=star_named_expressions '}' { _PyAST_Set(a, EXTRA) } +set[expr_ty]: '{' a=star_named_expressions_sequence '}' { _PyAST_Set(a, EXTRA) } # Dicts # ----- @@ -1035,20 +1055,20 @@ for_if_clause[comprehension_ty]: | invalid_for_target listcomp[expr_ty]: - | '[' a=named_expression b=for_if_clauses ']' { _PyAST_ListComp(a, b, EXTRA) } + | '[' a=star_named_expression b=for_if_clauses ']' { _PyAST_ListComp(a, b, EXTRA) } | invalid_comprehension setcomp[expr_ty]: - | '{' a=named_expression b=for_if_clauses '}' { _PyAST_SetComp(a, b, EXTRA) } + | '{' a=star_named_expression b=for_if_clauses '}' { _PyAST_SetComp(a, b, EXTRA) } | invalid_comprehension genexp[expr_ty]: - | '(' a=( assignment_expression | expression !':=') b=for_if_clauses ')' { _PyAST_GeneratorExp(a, b, EXTRA) } + | '(' a=( assignment_expression | expression !':=' | starred_expression ) b=for_if_clauses ')' { _PyAST_GeneratorExp(a, b, EXTRA) } | invalid_comprehension dictcomp[expr_ty]: | '{' a=kvpair b=for_if_clauses '}' { _PyAST_DictComp(a->key, a->value, b, EXTRA) } - | invalid_dict_comprehension + | '{' '**' a=expression b=for_if_clauses '}' { _PyAST_DictComp(a, NULL, b, EXTRA) } # FUNCTION CALL ARGUMENTS # ======================= @@ -1246,8 +1266,7 @@ invalid_expression: # !(NAME STRING) is not matched so we don't show this error with some invalid string prefixes like: kf"dsfsdf" # Soft keywords need to also be ignored because they can be parsed as NAME NAME | !(NAME STRING | SOFT_KEYWORD) a=disjunction b=expression_without_invalid { - _PyPegen_check_legacy_stmt(p, a) ? NULL : p->tokens[p->mark-1]->level == 0 ? NULL : - RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Perhaps you forgot a comma?") } + _PyPegen_raise_error_for_missing_comma(p, a, b) } | a=disjunction 'if' b=disjunction !('else'|':') { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "expected 'else' after 'if' expression") } | a=disjunction 'if' b=disjunction 'else' !expression { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("expected expression after 'else', but statement is given") } @@ -1258,6 +1277,12 @@ invalid_expression: | a='lambda' [lambda_params] b=':' &TSTRING_MIDDLE { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "t-string: lambda expressions are not allowed without parentheses") } +invalid_if_expression: + | disjunction 'if' b=disjunction 'else' a='*' { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot unpack only part of a conditional expression") } + | disjunction 'if' b=disjunction 'else' a='**' { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use dict unpacking on only part of a conditional expression") } + invalid_named_expression(memo): | a=expression ':=' expression { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( @@ -1300,19 +1325,37 @@ invalid_raise_stmt: invalid_del_stmt: | 'del' a=star_expressions { RAISE_SYNTAX_ERROR_INVALID_TARGET(DEL_TARGETS, a) } +invalid_assert_stmt: + | 'assert' a=expression '=' b=expression { + RAISE_SYNTAX_ERROR_KNOWN_RANGE( + a, b, + "cannot assign to %s here. Maybe you meant '==' instead of '='?", + _PyPegen_get_expr_name(a)) } + | 'assert' expression ',' a=expression '=' b=expression { + RAISE_SYNTAX_ERROR_KNOWN_RANGE( + a, b, + "cannot assign to %s here. Maybe you meant '==' instead of '='?", + _PyPegen_get_expr_name(a)) } + | 'assert' a=expression ':=' b=expression { + RAISE_SYNTAX_ERROR_KNOWN_RANGE( + a, b, + "cannot use named expression without parentheses here") } + | 'assert' expression ',' a=expression ':=' b=expression { + RAISE_SYNTAX_ERROR_KNOWN_RANGE( + a, b, + "cannot use named expression without parentheses here") } invalid_block: | NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block") } invalid_comprehension: - | ('[' | '(' | '{') a=starred_expression for_if_clauses { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "iterable unpacking cannot be used in comprehension") } + | '[' a='**' b=expression for_if_clauses { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in list comprehension") } + | '(' a='**' b=expression for_if_clauses { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in generator expression") } | ('[' | '{') a=star_named_expression ',' b=star_named_expressions for_if_clauses { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, PyPegen_last_item(b, expr_ty), "did you forget parentheses around the comprehension target?") } | ('[' | '{') a=star_named_expression b=',' for_if_clauses { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "did you forget parentheses around the comprehension target?") } -invalid_dict_comprehension: - | '{' a='**' bitwise_or for_if_clauses '}' { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "dict unpacking cannot be used in dict comprehension") } invalid_parameters: | a="/" ',' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "at least one parameter must precede /") } @@ -1392,11 +1435,11 @@ invalid_import: | 'import' token=NEWLINE { RAISE_SYNTAX_ERROR_STARTING_FROM(token, "Expected one or more names after 'import'") } invalid_dotted_as_name: - | dotted_name 'as' !(NAME (',' | ')' | NEWLINE)) a=expression { + | dotted_name 'as' !(NAME (',' | ')' | ';' | NEWLINE)) a=expression { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use %s as import target", _PyPegen_get_expr_name(a)) } invalid_import_from_as_name: - | NAME 'as' !(NAME (',' | ')' | NEWLINE)) a=expression { + | NAME 'as' !(NAME (',' | ')' | ';' | NEWLINE)) a=expression { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use %s as import target", _PyPegen_get_expr_name(a)) } @@ -1407,6 +1450,8 @@ invalid_import_from_targets: RAISE_SYNTAX_ERROR_STARTING_FROM(token, "Expected one or more names after 'import'") } invalid_with_stmt: + | ['async'] 'with' ','.(expression ['as' star_target])+ trailing=',' ':' { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(trailing, "the last 'with' item has a trailing comma") } | ['async'] 'with' ','.(expression ['as' star_target])+ NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | ['async'] 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } invalid_with_stmt_indent: @@ -1453,6 +1498,10 @@ invalid_match_stmt: | "match" subject_expr NEWLINE { CHECK_VERSION(void*, 10, "Pattern matching is", RAISE_SYNTAX_ERROR("expected ':'") ) } | a="match" subject=subject_expr ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after 'match' statement on line %d", a->lineno) } + | a="case" patterns guard? b=':' block { + RAISE_SYNTAX_ERROR_KNOWN_RANGE( + a, b, + "case statement must be inside match statement") } invalid_case_block: | "case" patterns guard? NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | a="case" patterns guard? ':' NEWLINE !INDENT { @@ -1467,6 +1516,10 @@ invalid_class_pattern: PyPegen_first_item(a, pattern_ty), PyPegen_last_item(a, pattern_ty), "positional patterns follow keyword patterns") } +invalid_mapping_pattern: + | '{' (items_pattern ',')? rest=double_star_pattern ',' items_pattern ','? '}' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + rest, + "double star pattern must be the last (right-most) subpattern in the mapping pattern") } invalid_class_argument_pattern[asdl_pattern_seq*]: | [positional_patterns ','] keyword_patterns ',' a=positional_patterns { a } invalid_if_stmt: @@ -1499,19 +1552,32 @@ invalid_class_def_raw: RAISE_INDENTATION_ERROR("expected an indented block after class definition on line %d", a->lineno) } invalid_double_starred_kvpairs: - | ','.double_starred_kvpair+ ',' invalid_kvpair - | expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use a starred expression in a dictionary value") } + | invalid_kvpair_unpacking [','] + | ','.double_starred_kvpair+ ',' (invalid_kvpair | invalid_kvpair_unpacking) | expression a=':' &('}'|',') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") } +invalid_kvpair_unpacking: + | a='**' b=if_expression { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid double starred expression. Did you forget to wrap the conditional expression in parentheses?") } + | a='*' b=bitwise_or ':' expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use a starred expression in a dictionary key") } + | a='**' b=bitwise_or ':' expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in a dictionary key") } + | expression ':' a='*' b=bitwise_or { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use a starred expression in a dictionary value") } + | expression ':' a='**' b=bitwise_or { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in a dictionary value") } invalid_kvpair: | a=expression !(':') { RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, a->lineno, a->end_col_offset - 1, a->end_lineno, -1, "':' expected after dictionary key") } | expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use a starred expression in a dictionary value") } + | expression ':' a='**' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use dict unpacking in a dictionary value") } | expression a=':' &('}'|',') {RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") } invalid_starred_expression_unpacking: + | a='*' b=if_expression { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid starred expression. Did you forget to wrap the conditional expression in parentheses?") } | a='*' expression '=' b=expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to iterable argument unpacking") } +invalid_starred_expression_unpacking_sequence: + | a='**' bitwise_or { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use dict unpacking here") } + | invalid_starred_expression_unpacking invalid_starred_expression: | '*' { RAISE_SYNTAX_ERROR("Invalid star expression") } - invalid_fstring_replacement_field: | '{' a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '='") } | '{' a='!' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '!'") } diff --git a/Include/Python.h b/Include/Python.h index 3f49b78947c9a63..337119c15fe8b6f 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -9,24 +9,27 @@ // is not needed. -// Include Python header files -#include "patchlevel.h" -#include "pyconfig.h" -#include "pymacconfig.h" +// Include Python configuration headers +#include "patchlevel.h" // the Python version +#include "pyconfig.h" // information from configure +#include "pymacconfig.h" // overrides for pyconfig +#include "pyabi.h" // feature/ABI selection // Include standard header files +// When changing these files, remember to update Doc/extending/extending.rst. #include // assert() #include // uintptr_t #include // INT_MAX #include // HUGE_VAL #include // va_list +#include // memcpy() #include // wchar_t #ifdef HAVE_SYS_TYPES_H # include // ssize_t #endif -// , , and headers are no longer used +// , and headers are no longer used // by Python, but kept for the backward compatibility of existing third party C // extensions. They are not included by limited C API version 3.11 and newer. // @@ -36,7 +39,6 @@ # include // errno # include // FILE* # include // getenv() -# include // memcpy() #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030d0000 # include // tolower() @@ -45,22 +47,26 @@ # endif #endif -#if defined(Py_GIL_DISABLED) -# if defined(Py_LIMITED_API) && !defined(_Py_OPAQUE_PYOBJECT) -# error "Py_LIMITED_API is not currently supported in the free-threaded build" +#if !defined(Py_LIMITED_API) +# if defined(Py_GIL_DISABLED) +# if defined(_MSC_VER) || defined(__MINGW32__) +# include // __readgsqword() +# endif # endif +#endif // Py_GIL_DISABLED -# if defined(_MSC_VER) -# include // __readgsqword() -# endif +#ifdef _MSC_VER +// Ignore MSC warning C4201: "nonstandard extension used: nameless +// struct/union". (Only generated for C standard versions less than C11, which +// we don't *officially* support.) +__pragma(warning(push)) +__pragma(warning(disable: 4201)) +#endif -# if defined(__MINGW32__) -# include // __readgsqword() -# endif -#endif // Py_GIL_DISABLED // Include Python header files #include "pyport.h" +#include "exports.h" #include "pymacro.h" #include "pymath.h" #include "pymem.h" @@ -68,12 +74,13 @@ #include "pybuffer.h" #include "pystats.h" #include "pyatomic.h" -#include "pylock.h" +#include "cpython/pylock.h" #include "critical_section.h" #include "object.h" #include "refcount.h" #include "objimpl.h" -#include "typeslots.h" +#include "slots.h" +#include "slots_generated.h" #include "pyhash.h" #include "cpython/pydebug.h" #include "bytearrayobject.h" @@ -95,7 +102,7 @@ #include "setobject.h" #include "methodobject.h" #include "moduleobject.h" -#include "monitoring.h" +#include "cpython/monitoring.h" #include "cpython/funcobject.h" #include "cpython/classobject.h" #include "fileobject.h" @@ -111,6 +118,7 @@ #include "cpython/genobject.h" #include "descrobject.h" #include "genericaliasobject.h" +#include "cpython/sentinelobject.h" #include "warnings.h" #include "weakrefobject.h" #include "structseq.h" @@ -138,4 +146,8 @@ #include "cpython/pyfpe.h" #include "cpython/tracemalloc.h" +#ifdef _MSC_VER +__pragma(warning(pop)) // warning(disable: 4201) +#endif + #endif /* !Py_PYTHON_H */ diff --git a/Include/cpython/abstract.h b/Include/cpython/abstract.h index ffd19ccd3500fac..7490ece52e5220e 100644 --- a/Include/cpython/abstract.h +++ b/Include/cpython/abstract.h @@ -6,7 +6,7 @@ /* Like PyObject_CallMethod(), but expect a _Py_Identifier* as the method name. */ -PyAPI_FUNC(PyObject*) _PyObject_CallMethodId( +Py_DEPRECATED(3.15) PyAPI_FUNC(PyObject*) _PyObject_CallMethodId( PyObject *obj, _Py_Identifier *name, const char *format, ...); diff --git a/Include/cpython/bytearrayobject.h b/Include/cpython/bytearrayobject.h index 4dddef713ce0970..1edd082074206cf 100644 --- a/Include/cpython/bytearrayobject.h +++ b/Include/cpython/bytearrayobject.h @@ -5,25 +5,25 @@ /* Object layout */ typedef struct { PyObject_VAR_HEAD - Py_ssize_t ob_alloc; /* How many bytes allocated in ob_bytes */ + /* How many bytes allocated in ob_bytes + + In the current implementation this is equivalent to Py_SIZE(ob_bytes_object). + The value is always loaded and stored atomically for thread safety. + There are API compatibilty concerns with removing so keeping for now. */ + Py_ssize_t ob_alloc; char *ob_bytes; /* Physical backing buffer */ char *ob_start; /* Logical start inside ob_bytes */ Py_ssize_t ob_exports; /* How many buffer exports */ + PyObject *ob_bytes_object; /* PyBytes for zero-copy bytes conversion */ } PyByteArrayObject; -PyAPI_DATA(char) _PyByteArray_empty_string[]; - /* Macros and static inline functions, trading safety for speed */ #define _PyByteArray_CAST(op) \ (assert(PyByteArray_Check(op)), _Py_CAST(PyByteArrayObject*, op)) static inline char* PyByteArray_AS_STRING(PyObject *op) { - PyByteArrayObject *self = _PyByteArray_CAST(op); - if (Py_SIZE(self)) { - return self->ob_start; - } - return _PyByteArray_empty_string; + return _PyByteArray_CAST(op)->ob_start; } #define PyByteArray_AS_STRING(self) PyByteArray_AS_STRING(_PyObject_CAST(self)) diff --git a/Include/cpython/bytesobject.h b/Include/cpython/bytesobject.h index 71c133f173f157f..85bc2b827df8fb0 100644 --- a/Include/cpython/bytesobject.h +++ b/Include/cpython/bytesobject.h @@ -40,3 +40,46 @@ _PyBytes_Join(PyObject *sep, PyObject *iterable) { return PyBytes_Join(sep, iterable); } + + +// --- PyBytesWriter API ----------------------------------------------------- + +typedef struct PyBytesWriter PyBytesWriter; + +PyAPI_FUNC(PyBytesWriter *) PyBytesWriter_Create( + Py_ssize_t size); +PyAPI_FUNC(void) PyBytesWriter_Discard( + PyBytesWriter *writer); +PyAPI_FUNC(PyObject*) PyBytesWriter_Finish( + PyBytesWriter *writer); +PyAPI_FUNC(PyObject*) PyBytesWriter_FinishWithSize( + PyBytesWriter *writer, + Py_ssize_t size); +PyAPI_FUNC(PyObject*) PyBytesWriter_FinishWithPointer( + PyBytesWriter *writer, + void *buf); + +PyAPI_FUNC(void*) PyBytesWriter_GetData( + PyBytesWriter *writer); +PyAPI_FUNC(Py_ssize_t) PyBytesWriter_GetSize( + PyBytesWriter *writer); + +PyAPI_FUNC(int) PyBytesWriter_WriteBytes( + PyBytesWriter *writer, + const void *bytes, + Py_ssize_t size); +PyAPI_FUNC(int) PyBytesWriter_Format( + PyBytesWriter *writer, + const char *format, + ...); + +PyAPI_FUNC(int) PyBytesWriter_Resize( + PyBytesWriter *writer, + Py_ssize_t size); +PyAPI_FUNC(int) PyBytesWriter_Grow( + PyBytesWriter *writer, + Py_ssize_t size); +PyAPI_FUNC(void*) PyBytesWriter_GrowAndUpdatePointer( + PyBytesWriter *writer, + Py_ssize_t size, + void *buf); diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index ca8109e3248a8d8..5b66fa1040d738f 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -23,6 +23,9 @@ _PyEval_RequestCodeExtraIndex(freefunc f) { PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *); PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *); +PyAPI_FUNC(int) _PyEval_UnpackIndices(PyObject *, PyObject *, + Py_ssize_t, + Py_ssize_t *, Py_ssize_t *); // Trampoline API @@ -35,7 +38,7 @@ typedef struct { PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void); PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( const void *code_addr, - unsigned int code_size, + size_t code_size, const char *entry_name); PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void); PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename); diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 3f0dce034555260..84456a709a6abe7 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -282,15 +282,6 @@ typedef struct _line_offsets { */ PyAPI_FUNC(int) _PyCode_CheckLineNumber(int lasti, PyCodeAddressRange *bounds); -/* Create a comparable key used to compare constants taking in account the - * object type. It is used to make sure types are not coerced (e.g., float and - * complex) _and_ to distinguish 0.0 from -0.0 e.g. on IEEE platforms - * - * Return (type(obj), obj, ...): a tuple with variable size (at least 2 items) - * depending on the type and the value. The type is the first item to not - * compare bytes and str which can raise a BytesWarning exception. */ -PyAPI_FUNC(PyObject*) _PyCode_ConstantKey(PyObject *obj); - PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, PyObject *lnotab); diff --git a/Include/cpython/complexobject.h b/Include/cpython/complexobject.h index fbdc6a91fe895c0..58da80140dc4c99 100644 --- a/Include/cpython/complexobject.h +++ b/Include/cpython/complexobject.h @@ -7,7 +7,8 @@ typedef struct { double imag; } Py_complex; -// Operations on complex numbers. +/* Operations on complex numbers (soft deprecated + since Python 3.15). */ PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_diff(Py_complex, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_neg(Py_complex); diff --git a/Include/cpython/critical_section.h b/Include/cpython/critical_section.h index 4fc46fefb93a24a..bcba32da412f326 100644 --- a/Include/cpython/critical_section.h +++ b/Include/cpython/critical_section.h @@ -2,15 +2,6 @@ # error "this header file must not be included directly" #endif -// Python critical sections -// -// Conceptually, critical sections are a deadlock avoidance layer on top of -// per-object locks. These helpers, in combination with those locks, replace -// our usage of the global interpreter lock to provide thread-safety for -// otherwise thread-unsafe objects, such as dict. -// -// NOTE: These APIs are no-ops in non-free-threaded builds. -// // Straightforward per-object locking could introduce deadlocks that were not // present when running with the GIL. Threads may hold locks for multiple // objects simultaneously because Python operations can nest. If threads were @@ -43,52 +34,19 @@ // `_PyThreadState_Attach()`, it resumes the top-most (i.e., most recent) // critical section by reacquiring the associated lock or locks. See // `_PyCriticalSection_Resume()`. -// -// NOTE: Only the top-most critical section is guaranteed to be active. -// Operations that need to lock two objects at once must use -// `Py_BEGIN_CRITICAL_SECTION2()`. You *CANNOT* use nested critical sections -// to lock more than one object at once, because the inner critical section -// may suspend the outer critical sections. This API does not provide a way -// to lock more than two objects at once (though it could be added later -// if actually needed). -// -// NOTE: Critical sections implicitly behave like reentrant locks because -// attempting to acquire the same lock will suspend any outer (earlier) -// critical sections. However, they are less efficient for this use case than -// purposefully designed reentrant locks. -// -// Example usage: -// Py_BEGIN_CRITICAL_SECTION(op); -// ... -// Py_END_CRITICAL_SECTION(); -// -// To lock two objects at once: -// Py_BEGIN_CRITICAL_SECTION2(op1, op2); -// ... -// Py_END_CRITICAL_SECTION2(); - -typedef struct PyCriticalSection PyCriticalSection; -typedef struct PyCriticalSection2 PyCriticalSection2; - -PyAPI_FUNC(void) -PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op); PyAPI_FUNC(void) PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m); -PyAPI_FUNC(void) -PyCriticalSection_End(PyCriticalSection *c); - -PyAPI_FUNC(void) -PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b); - PyAPI_FUNC(void) PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2); -PyAPI_FUNC(void) -PyCriticalSection2_End(PyCriticalSection2 *c); - #ifndef Py_GIL_DISABLED +#undef Py_BEGIN_CRITICAL_SECTION +#undef Py_END_CRITICAL_SECTION +#undef Py_BEGIN_CRITICAL_SECTION2 +#undef Py_END_CRITICAL_SECTION2 + # define Py_BEGIN_CRITICAL_SECTION(op) \ { # define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ @@ -101,54 +59,17 @@ PyCriticalSection2_End(PyCriticalSection2 *c); { # define Py_END_CRITICAL_SECTION2() \ } -#else /* !Py_GIL_DISABLED */ - -// NOTE: the contents of this struct are private and may change betweeen -// Python releases without a deprecation period. -struct PyCriticalSection { - // Tagged pointer to an outer active critical section (or 0). - uintptr_t _cs_prev; - - // Mutex used to protect critical section - PyMutex *_cs_mutex; -}; - -// A critical section protected by two mutexes. Use -// Py_BEGIN_CRITICAL_SECTION2 and Py_END_CRITICAL_SECTION2. -// NOTE: the contents of this struct are private and may change betweeen -// Python releases without a deprecation period. -struct PyCriticalSection2 { - PyCriticalSection _cs_base; - PyMutex *_cs_mutex2; -}; - -# define Py_BEGIN_CRITICAL_SECTION(op) \ - { \ - PyCriticalSection _py_cs; \ - PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op)) +#else /* !Py_GIL_DISABLED */ # define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ { \ PyCriticalSection _py_cs; \ PyCriticalSection_BeginMutex(&_py_cs, mutex) -# define Py_END_CRITICAL_SECTION() \ - PyCriticalSection_End(&_py_cs); \ - } - -# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ - { \ - PyCriticalSection2 _py_cs2; \ - PyCriticalSection2_Begin(&_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) - # define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) \ { \ PyCriticalSection2 _py_cs2; \ PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) -# define Py_END_CRITICAL_SECTION2() \ - PyCriticalSection2_End(&_py_cs2); \ - } - -#endif +#endif /* !Py_GIL_DISABLED */ diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index df9ec7050fca1a9..5e7811416aba63f 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -32,6 +32,16 @@ typedef struct { PyDictValues *ma_values; } PyDictObject; +// frozendict +PyAPI_DATA(PyTypeObject) PyFrozenDict_Type; +#define PyFrozenDict_Check(op) PyObject_TypeCheck((op), &PyFrozenDict_Type) +#define PyFrozenDict_CheckExact(op) Py_IS_TYPE((op), &PyFrozenDict_Type) + +#define PyAnyDict_CheckExact(ob) \ + (PyDict_CheckExact(ob) || PyFrozenDict_CheckExact(ob)) +#define PyAnyDict_Check(ob) \ + (PyDict_Check(ob) || PyFrozenDict_Check(ob)) + PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, Py_hash_t hash); // PyDict_GetItemStringRef() can be used instead @@ -39,20 +49,10 @@ Py_DEPRECATED(3.14) PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObje PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); -// Inserts `key` with a value `default_value`, if `key` is not already present -// in the dictionary. If `result` is not NULL, then the value associated -// with `key` is returned in `*result` (either the existing value, or the now -// inserted `default_value`). -// Returns: -// -1 on error -// 0 if `key` was not present and `default_value` was inserted -// 1 if `key` was present and `default_value` was not inserted -PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); - /* Get the number of items of a dictionary. */ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyDictObject *mp; - assert(PyDict_Check(op)); + assert(PyAnyDict_Check(op)); mp = _Py_CAST(PyDictObject*, op); #ifdef Py_GIL_DISABLED return _Py_atomic_load_ssize_relaxed(&mp->ma_used); @@ -103,3 +103,6 @@ PyAPI_FUNC(int) PyDict_ClearWatcher(int watcher_id); // Mark given dictionary as "watched" (callback will be called if it is modified) PyAPI_FUNC(int) PyDict_Watch(int watcher_id, PyObject* dict); PyAPI_FUNC(int) PyDict_Unwatch(int watcher_id, PyObject* dict); + +// Create a frozendict. Create an empty dictionary if iterable is NULL. +PyAPI_FUNC(PyObject*) PyFrozenDict_New(PyObject *iterable); diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index 598cd330bc9ca95..9e1599a76485646 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -134,7 +134,8 @@ PyAPI_FUNC(PyObject *) PyStaticMethod_New(PyObject *); V(DESTROY) \ V(MODIFY_CODE) \ V(MODIFY_DEFAULTS) \ - V(MODIFY_KWDEFAULTS) + V(MODIFY_KWDEFAULTS) \ + V(MODIFY_QUALNAME) typedef enum { #define PY_DEF_EVENT(EVENT) PyFunction_EVENT_##EVENT, diff --git a/Include/cpython/genobject.h b/Include/cpython/genobject.h index f75884e597e2c24..e14facd77edec53 100644 --- a/Include/cpython/genobject.h +++ b/Include/cpython/genobject.h @@ -16,8 +16,8 @@ PyAPI_DATA(PyTypeObject) PyGen_Type; #define PyGen_Check(op) PyObject_TypeCheck((op), &PyGen_Type) #define PyGen_CheckExact(op) Py_IS_TYPE((op), &PyGen_Type) -PyAPI_FUNC(PyObject *) PyGen_New(PyFrameObject *); -PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(PyFrameObject *, +Py_DEPRECATED(3.16) PyAPI_FUNC(PyObject *) PyGen_New(PyFrameObject *); +Py_DEPRECATED(3.16) PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(PyFrameObject *, PyObject *name, PyObject *qualname); PyAPI_FUNC(PyCodeObject *) PyGen_GetCode(PyGenObject *gen); @@ -29,7 +29,7 @@ typedef struct _PyCoroObject PyCoroObject; PyAPI_DATA(PyTypeObject) PyCoro_Type; #define PyCoro_CheckExact(op) Py_IS_TYPE((op), &PyCoro_Type) -PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, +Py_DEPRECATED(3.16) PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, PyObject *name, PyObject *qualname); @@ -40,7 +40,7 @@ typedef struct _PyAsyncGenObject PyAsyncGenObject; PyAPI_DATA(PyTypeObject) PyAsyncGen_Type; PyAPI_DATA(PyTypeObject) _PyAsyncGenASend_Type; -PyAPI_FUNC(PyObject *) PyAsyncGen_New(PyFrameObject *, +Py_DEPRECATED(3.16) PyAPI_FUNC(PyObject *) PyAsyncGen_New(PyFrameObject *, PyObject *name, PyObject *qualname); #define PyAsyncGen_CheckExact(op) Py_IS_TYPE((op), &PyAsyncGen_Type) diff --git a/Include/cpython/import.h b/Include/cpython/import.h index 0ce0b1ee6cce2a2..149a20af8b9cbb2 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -10,6 +10,13 @@ struct _inittab { PyAPI_DATA(struct _inittab *) PyImport_Inittab; PyAPI_FUNC(int) PyImport_ExtendInittab(struct _inittab *newtab); +// Custom importers may use this API to initialize statically linked +// extension modules directly from a spec and init function, +// without needing to go through inittab +PyAPI_FUNC(PyObject *) PyImport_CreateModuleFromInitfunc( + PyObject *spec, + PyObject *(*initfunc)(void)); + struct _frozen { const char *name; /* ASCII encoded string */ const unsigned char *code; diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 1c979d91a40850b..1ccc496c63ac780 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -149,6 +149,7 @@ typedef struct PyConfig { int dump_refs; wchar_t *dump_refs_file; int malloc_stats; + int pymalloc_hugepages; wchar_t *filesystem_encoding; wchar_t *filesystem_errors; wchar_t *pycache_prefix; @@ -190,6 +191,7 @@ typedef struct PyConfig { int enable_gil; int tlbc_enabled; #endif + int lazy_imports; /* --- Path configuration inputs ------------ */ int pathconfig_warnings; diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index 4b6f97a5e475d65..998ebe6891577e8 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -133,50 +133,16 @@ _PyLong_CompactValue(const PyLongObject *op) assert(PyType_HasFeature(op->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); assert(PyUnstable_Long_IsCompact(op)); sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); + if (sign == 0) { + // gh-147988: Make sure that the digit is zero. + // It helps detecting the usage of uninitialized digits. + assert(op->long_value.ob_digit[0] == 0); + } return sign * (Py_ssize_t)op->long_value.ob_digit[0]; } #define PyUnstable_Long_CompactValue _PyLong_CompactValue - -/* --- Import/Export API -------------------------------------------------- */ - -typedef struct PyLongLayout { - uint8_t bits_per_digit; - uint8_t digit_size; - int8_t digits_order; - int8_t digit_endianness; -} PyLongLayout; - -PyAPI_FUNC(const PyLongLayout*) PyLong_GetNativeLayout(void); - -typedef struct PyLongExport { - int64_t value; - uint8_t negative; - Py_ssize_t ndigits; - const void *digits; - // Member used internally, must not be used for other purpose. - Py_uintptr_t _reserved; -} PyLongExport; - -PyAPI_FUNC(int) PyLong_Export( - PyObject *obj, - PyLongExport *export_long); -PyAPI_FUNC(void) PyLong_FreeExport( - PyLongExport *export_long); - - -/* --- PyLongWriter API --------------------------------------------------- */ - -typedef struct PyLongWriter PyLongWriter; - -PyAPI_FUNC(PyLongWriter*) PyLongWriter_Create( - int negative, - Py_ssize_t ndigits, - void **digits); -PyAPI_FUNC(PyObject*) PyLongWriter_Finish(PyLongWriter *writer); -PyAPI_FUNC(void) PyLongWriter_Discard(PyLongWriter *writer); - #ifdef __cplusplus } #endif diff --git a/Include/cpython/marshal.h b/Include/cpython/marshal.h new file mode 100644 index 000000000000000..159459fcaec3d93 --- /dev/null +++ b/Include/cpython/marshal.h @@ -0,0 +1,17 @@ +#ifndef _Py_CPYTHON_MARSHAL_H +# error "this header file must not be included directly" +#endif + +PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, + Py_ssize_t); +PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int); + +#define Py_MARSHAL_VERSION 6 + +PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *); +PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *); +PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromFile(FILE *); +PyAPI_FUNC(PyObject *) PyMarshal_ReadLastObjectFromFile(FILE *); + +PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int); +PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int); diff --git a/Include/cpython/modsupport.h b/Include/cpython/modsupport.h index d3b88f58c82ca3e..cfeee6e8ab3414d 100644 --- a/Include/cpython/modsupport.h +++ b/Include/cpython/modsupport.h @@ -2,6 +2,19 @@ # error "this header file must not be included directly" #endif +PyAPI_FUNC(int) PyArg_ParseArray( + PyObject *const *args, + Py_ssize_t nargs, + const char *format, + ...); +PyAPI_FUNC(int) PyArg_ParseArrayAndKeywords( + PyObject *const *args, + Py_ssize_t nargs, + PyObject *kwnames, + const char *format, + const char * const *kwlist, + ...); + // A data structure that can be used to run initialization code once in a // thread-safe manner. The C++11 equivalent is std::call_once. typedef struct { @@ -24,3 +37,9 @@ typedef struct _PyArg_Parser { PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast(PyObject *, PyObject *, struct _PyArg_Parser *, ...); + +#ifdef Py_BUILD_CORE +// For internal use in stdlib. Needs C99 compound literals. +// Defined here to avoid every stdlib module including pycore_modsupport.h +#define _Py_ABI_SLOT {Py_mod_abi, (void*) &(PyABIInfo) _PyABIInfo_DEFAULT} +#endif diff --git a/Include/cpython/monitoring.h b/Include/cpython/monitoring.h index ce92942404c9f7d..c93271f6ca95f5b 100644 --- a/Include/cpython/monitoring.h +++ b/Include/cpython/monitoring.h @@ -1,7 +1,13 @@ -#ifndef Py_CPYTHON_MONITORING_H -# error "this header file must not be included directly" +#ifndef Py_MONITORING_H +#define Py_MONITORING_H +#ifndef Py_LIMITED_API +#ifdef __cplusplus +extern "C" { #endif +// There is currently no limited API for monitoring + + /* Local events. * These require bytecode instrumentation */ @@ -18,9 +24,10 @@ #define PY_MONITORING_EVENT_STOP_ITERATION 10 #define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ - ((ev) < _PY_MONITORING_LOCAL_EVENTS) +((ev) <= PY_MONITORING_EVENT_STOP_ITERATION) -/* Other events, mainly exceptions */ +/* Other events, mainly exceptions. + * These can now be turned on and disabled on a per code object basis. */ #define PY_MONITORING_EVENT_RAISE 11 #define PY_MONITORING_EVENT_EXCEPTION_HANDLED 12 @@ -28,6 +35,9 @@ #define PY_MONITORING_EVENT_PY_THROW 14 #define PY_MONITORING_EVENT_RERAISE 15 +#define _PY_MONITORING_IS_UNGROUPED_EVENT(ev) \ +((ev) < _PY_MONITORING_UNGROUPED_EVENTS) + /* Ancillary events */ @@ -267,3 +277,9 @@ PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike } #undef _PYMONITORING_IF_ACTIVE + +#ifdef __cplusplus +} +#endif +#endif // !Py_LIMITED_API +#endif // !Py_MONITORING_H diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 973d358ed8e4ec7..326254c335b4895 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -230,6 +230,8 @@ struct _typeobject { destructor tp_finalize; vectorcallfunc tp_vectorcall; + /* Below here all fields are internal to the VM */ + /* bitset of which type-watchers care about this type */ unsigned char tp_watched; @@ -239,6 +241,11 @@ struct _typeobject { * Otherwise, limited to MAX_VERSIONS_PER_CLASS (defined elsewhere). */ uint16_t tp_versions_used; + + /* Virtual iterator next function. + * This function must escape to any code that can result in + * the GC being run, such as Py_DECREF. */ + _Py_iteritemfunc _tp_iteritem; }; #define _Py_ATTR_CACHE_UNUSED (30000) // (see tp_versions_used) @@ -295,13 +302,15 @@ PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); PyAPI_FUNC(void) _Py_BreakPoint(void); -PyAPI_FUNC(void) _PyObject_Dump(PyObject *); +PyAPI_FUNC(void) PyObject_Dump(PyObject *); + +// Alias for backward compatibility +#define _PyObject_Dump PyObject_Dump -PyAPI_FUNC(PyObject*) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); +Py_DEPRECATED(3.15) PyAPI_FUNC(PyObject*) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); PyAPI_FUNC(PyObject **) _PyObject_GetDictPtr(PyObject *); PyAPI_FUNC(void) PyObject_CallFinalizer(PyObject *); -PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *); PyAPI_FUNC(void) PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *); @@ -387,10 +396,11 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); process with a message on stderr if the given condition fails to hold, but compile away to nothing if NDEBUG is defined. - However, before aborting, Python will also try to call _PyObject_Dump() on - the given object. This may be of use when investigating bugs in which a - particular object is corrupt (e.g. buggy a tp_visit method in an extension - module breaking the garbage collector), to help locate the broken objects. + However, before aborting, Python will also try to call + PyObject_Dump() on the given object. This may be of use when + investigating bugs in which a particular object is corrupt (e.g. buggy a + tp_visit method in an extension module breaking the garbage collector), to + help locate the broken objects. The WITH_MSG variant allows you to supply an additional message that Python will attempt to print to stderr, after the object dump. */ @@ -432,17 +442,15 @@ PyAPI_FUNC(void) _Py_NO_RETURN _PyObject_AssertFailed( PyAPI_FUNC(void) _PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op); PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(PyThreadState *tstate); -PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count); - /* For backwards compatibility with the old trashcan mechanism */ #define Py_TRASHCAN_BEGIN(op, dealloc) #define Py_TRASHCAN_END PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj); +PyAPI_FUNC(void *) PyObject_GetItemData_DuringGC(PyObject *obj); PyAPI_FUNC(int) PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); -PyAPI_FUNC(int) _PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict); PyAPI_FUNC(void) PyObject_ClearManagedDict(PyObject *obj); @@ -463,6 +471,7 @@ PyAPI_FUNC(int) PyUnstable_Type_AssignVersionTag(PyTypeObject *type); typedef enum { PyRefTracer_CREATE = 0, PyRefTracer_DESTROY = 1, + PyRefTracer_TRACKER_REMOVED = 2, } PyRefTracerEvent; typedef int (*PyRefTracer)(PyObject *, PyRefTracerEvent event, void *); @@ -491,3 +500,84 @@ PyAPI_FUNC(int) PyUnstable_TryIncRef(PyObject *); PyAPI_FUNC(void) PyUnstable_EnableTryIncRef(PyObject *); PyAPI_FUNC(int) PyUnstable_Object_IsUniquelyReferenced(PyObject *); + +PyAPI_FUNC(int) PyUnstable_SetImmortal(PyObject *op); + +#if defined(Py_GIL_DISABLED) +PyAPI_FUNC(uintptr_t) _Py_GetThreadLocal_Addr(void); + +static inline uintptr_t +_Py_ThreadId(void) +{ + uintptr_t tid; +#if defined(_MSC_VER) && defined(_M_X64) + tid = __readgsqword(48); +#elif defined(_MSC_VER) && defined(_M_IX86) + tid = __readfsdword(24); +#elif defined(_MSC_VER) && defined(_M_ARM64) + tid = __getReg(18); +#elif defined(__MINGW32__) && defined(_M_X64) + tid = __readgsqword(48); +#elif defined(__MINGW32__) && defined(_M_IX86) + tid = __readfsdword(24); +#elif defined(__MINGW32__) && defined(_M_ARM64) + tid = __getReg(18); +#elif defined(__i386__) + __asm__("{movl %%gs:0, %0|mov %0, dword ptr gs:[0]}" : "=r" (tid)); // 32-bit always uses GS +#elif defined(__MACH__) && defined(__x86_64__) + __asm__("{movq %%gs:0, %0|mov %0, qword ptr gs:[0]}" : "=r" (tid)); // x86_64 macOSX uses GS +#elif defined(__x86_64__) + __asm__("{movq %%fs:0, %0|mov %0, qword ptr fs:[0]}" : "=r" (tid)); // x86_64 Linux, BSD uses FS +#elif defined(__arm__) && __ARM_ARCH >= 7 + __asm__ ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tid)); +#elif defined(__aarch64__) && defined(__APPLE__) + __asm__ ("mrs %0, tpidrro_el0" : "=r" (tid)); +#elif defined(__aarch64__) + __asm__ ("mrs %0, tpidr_el0" : "=r" (tid)); +#elif defined(__powerpc64__) + #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) + tid = (uintptr_t)__builtin_thread_pointer(); + #else + // r13 is reserved for use as system thread ID by the Power 64-bit ABI. + register uintptr_t tp __asm__ ("r13"); + __asm__("" : "=r" (tp)); + tid = tp; + #endif +#elif defined(__powerpc__) + #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) + tid = (uintptr_t)__builtin_thread_pointer(); + #else + // r2 is reserved for use as system thread ID by the Power 32-bit ABI. + register uintptr_t tp __asm__ ("r2"); + __asm__ ("" : "=r" (tp)); + tid = tp; + #endif +#elif defined(__s390__) && defined(__GNUC__) + // Both GCC and Clang have supported __builtin_thread_pointer + // for s390 from long time ago. + tid = (uintptr_t)__builtin_thread_pointer(); +#elif defined(__riscv) + #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) + tid = (uintptr_t)__builtin_thread_pointer(); + #else + // tp is Thread Pointer provided by the RISC-V ABI. + __asm__ ("mv %0, tp" : "=r" (tid)); + #endif +#else + // Fallback to a portable implementation if we do not have a faster + // platform-specific implementation. + tid = _Py_GetThreadLocal_Addr(); +#endif + return tid; +} + +static inline Py_ALWAYS_INLINE int +_Py_IsOwnedByCurrentThread(PyObject *ob) +{ +#ifdef _Py_THREAD_SANITIZER + return _Py_atomic_load_uintptr_relaxed(&ob->ob_tid) == _Py_ThreadId(); +#else + return ob->ob_tid == _Py_ThreadId(); +#endif +} +#endif diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 2a0c11e7b3ad665..e85b360c986668c 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -72,8 +72,8 @@ // def _Py_atomic_load_ptr_acquire(obj): // return obj # acquire // -// def _Py_atomic_store_ptr_release(obj): -// return obj # release +// def _Py_atomic_store_ptr_release(obj, value): +// obj = value # release // // def _Py_atomic_fence_seq_cst(): // # sequential consistency @@ -523,12 +523,18 @@ _Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value); static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); +static inline void +_Py_atomic_store_int8_release(int8_t *obj, int8_t value); + static inline void _Py_atomic_store_int_release(int *obj, int value); static inline int _Py_atomic_load_int_acquire(const int *obj); +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value); + static inline void _Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value); @@ -591,6 +597,17 @@ static inline void _Py_atomic_fence_release(void); // --- aliases --------------------------------------------------------------- +// Compilers don't really support "consume" semantics, so we fake it. Use +// "acquire" with TSan to support false positives. Use "relaxed" otherwise, +// because CPUs on all platforms we support respect address dependencies without +// extra barriers. +// See 2.6.7 in https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2055r0.pdf +#if defined(_Py_THREAD_SANITIZER) +# define _Py_atomic_load_ptr_consume _Py_atomic_load_ptr_acquire +#else +# define _Py_atomic_load_ptr_consume _Py_atomic_load_ptr_relaxed +#endif + #if SIZEOF_LONG == 8 # define _Py_atomic_load_ulong(p) \ _Py_atomic_load_uint64((uint64_t *)p) diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 1566b83b9f6a1b6..253b35082aafcd2 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -572,10 +572,18 @@ static inline void _Py_atomic_store_int_release(int *obj, int value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_int8_release(int8_t *obj, int8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline int _Py_atomic_load_int_acquire(const int *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index d155955df0cddf6..3b3c5f7017e9575 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -971,12 +971,6 @@ _Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) *(volatile unsigned short *)obj = value; } -static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) -{ - *(volatile unsigned int *)obj = value; -} - static inline void _Py_atomic_store_long_relaxed(long *obj, long value) { @@ -1066,6 +1060,32 @@ _Py_atomic_store_int_release(int *obj, int value) #endif } +static inline void +_Py_atomic_store_int8_release(int8_t *obj, int8_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(int8_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int8); + __stlr8((unsigned __int8 volatile *)obj, (unsigned __int8)value); +#else +# error "no implementation of _Py_atomic_store_int8_release" +#endif +} + +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(volatile unsigned int *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_uint_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 69a8b9e615ea5ff..faef303da70314c 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -459,7 +459,7 @@ static inline uint16_t _Py_atomic_load_uint16(const uint16_t *obj) { _Py_USING_STD; - return atomic_load((const _Atomic(uint32_t)*)obj); + return atomic_load((const _Atomic(uint16_t)*)obj); } static inline uint32_t @@ -948,14 +948,6 @@ _Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) memory_order_relaxed); } -static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) -{ - _Py_USING_STD; - atomic_store_explicit((_Atomic(unsigned int)*)obj, value, - memory_order_relaxed); -} - static inline void _Py_atomic_store_long_relaxed(long *obj, long value) { @@ -1031,6 +1023,22 @@ _Py_atomic_store_int_release(int *obj, int value) memory_order_release); } +static inline void +_Py_atomic_store_int8_release(int8_t *obj, int8_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int8_t)*)obj, value, + memory_order_release); +} + +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 6b63d304b0d9297..be2e3b641c25cb6 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -18,6 +18,7 @@ typedef struct { PyException_HEAD PyObject *msg; PyObject *excs; + PyObject *excs_str; } PyBaseExceptionGroupObject; typedef struct { diff --git a/Include/cpython/pyhash.h b/Include/cpython/pyhash.h index a33ba10b8d3a37c..dac223368dbe5e4 100644 --- a/Include/cpython/pyhash.h +++ b/Include/cpython/pyhash.h @@ -7,7 +7,7 @@ /* Parameters used for the numeric hash implementation. See notes for _Py_HashDouble in Python/pyhash.c. Numeric hashes are based on - reduction modulo the prime 2**_PyHASH_BITS - 1. */ + reduction modulo the prime 2**PyHASH_BITS - 1. */ #if SIZEOF_VOID_P >= 8 # define PyHASH_BITS 61 @@ -15,7 +15,7 @@ # define PyHASH_BITS 31 #endif -#define PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) +#define PyHASH_MODULUS (((size_t)1 << PyHASH_BITS) - 1) #define PyHASH_INF 314159 #define PyHASH_IMAG PyHASH_MULTIPLIER diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index 86ce6e6f79824a7..e46dfe59ec46304 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -25,6 +25,9 @@ PyAPI_FUNC(PyStatus) Py_PreInitializeFromArgs( PyAPI_FUNC(PyStatus) Py_InitializeFromConfig( const PyConfig *config); +// Python 3.8 provisional API (PEP 587) +PyAPI_FUNC(PyStatus) _Py_InitializeMain(void); + PyAPI_FUNC(int) Py_RunMain(void); diff --git a/Include/cpython/pylock.h b/Include/cpython/pylock.h index 63886fca28eae24..460ac2c9f80e81b 100644 --- a/Include/cpython/pylock.h +++ b/Include/cpython/pylock.h @@ -1,7 +1,11 @@ -#ifndef Py_CPYTHON_LOCK_H -# error "this header file must not be included directly" +#ifndef Py_LOCK_H +#define Py_LOCK_H +#ifndef Py_LIMITED_API +#ifdef __cplusplus +extern "C" { #endif + #define _Py_UNLOCKED 0 #define _Py_LOCKED 1 @@ -72,3 +76,10 @@ _PyMutex_IsLocked(PyMutex *m) return (_Py_atomic_load_uint8(&m->_bits) & _Py_LOCKED) != 0; } #define PyMutex_IsLocked _PyMutex_IsLocked + + +#ifdef __cplusplus +} +#endif +#endif // !Py_LIMITED_API +#endif // !Py_LOCK_H diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index be582122118e44e..a9d97e47e005dff 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -28,10 +28,10 @@ typedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *); #define PyTrace_OPCODE 7 /* Remote debugger support */ -#define Py_MAX_SCRIPT_PATH_SIZE 512 +#define _Py_MAX_SCRIPT_PATH_SIZE 512 typedef struct { int32_t debugger_pending_call; - char debugger_script_path[Py_MAX_SCRIPT_PATH_SIZE]; + char debugger_script_path[_Py_MAX_SCRIPT_PATH_SIZE]; } _PyRemoteDebuggerSupport; typedef struct _err_stackitem { @@ -105,13 +105,17 @@ struct _ts { # define _PyThreadState_WHENCE_INIT 1 # define _PyThreadState_WHENCE_FINI 2 # define _PyThreadState_WHENCE_THREADING 3 -# define _PyThreadState_WHENCE_GILSTATE 4 +# define _PyThreadState_WHENCE_C_API 4 # define _PyThreadState_WHENCE_EXEC 5 +# define _PyThreadState_WHENCE_THREADING_DAEMON 6 #endif /* Currently holds the GIL. Must be its own field to avoid data races */ int holds_gil; + /* Currently requesting the GIL */ + int gil_requested; + int _whence; /* Thread state (_Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, _Py_THREAD_SUSPENDED). @@ -131,6 +135,15 @@ struct _ts { /* Pointer to currently executing frame. */ struct _PyInterpreterFrame *current_frame; + /* Pointer to the base frame (bottommost sentinel frame). + Used by profilers to validate complete stack unwinding. + Points to the embedded base_frame in _PyThreadStateImpl. + The frame is embedded there rather than here because _PyInterpreterFrame + is defined in internal headers that cannot be exposed in the public API. */ + struct _PyInterpreterFrame *base_frame; + + struct _PyInterpreterFrame *last_profiled_frame; + Py_tracefunc c_profilefunc; Py_tracefunc c_tracefunc; PyObject *c_profileobj; @@ -157,6 +170,11 @@ struct _ts { */ unsigned long native_thread_id; + /* List of objects that still need to be cleaned up, singly linked + * via their gc headers' gc_next pointers. The list is populated by + * _PyTrash_thread_deposit_object and cleaned up by + * _PyTrash_thread_destroy_chain. + */ PyObject *delete_later; /* Tagged pointer to top-most critical section, or zero if there is no @@ -180,6 +198,7 @@ struct _ts { _PyStackChunk *datastack_chunk; PyObject **datastack_top; PyObject **datastack_limit; + _PyStackChunk *datastack_cached_chunk; /* XXX signal handlers should also be here */ /* The following fields are here to avoid allocation during init. @@ -198,6 +217,9 @@ struct _ts { PyObject *current_executor; + /* Internal to the JIT */ + struct _PyExitData *jit_exit; + uint64_t dict_global_version; /* Used to store/retrieve `threading.local` keys/values for this thread */ @@ -208,6 +230,29 @@ struct _ts { */ PyObject *threading_local_sentinel; _PyRemoteDebuggerSupport remote_debugger_support; + +#ifdef Py_STATS + // Pointer to PyStats structure, NULL if recording is off. For the + // free-threaded build, the structure is per-thread (stored as a pointer + // in _PyThreadStateImpl). For the default build, the structure is stored + // in the PyInterpreterState structure (threads do not have their own + // structure and all share the same per-interpreter structure). + PyStats *pystats; +#endif + + struct { + /* Number of nested PyThreadState_Ensure() calls on this thread state */ + Py_ssize_t counter; + + /* Should this thread state be deleted upon calling + PyThreadState_Release() (with the counter at 1)? + + This is only true for thread states created by PyThreadState_Ensure() */ + int delete_on_release; + + /* The interpreter guard owned by PyThreadState_EnsureFromView(), if any. */ + PyInterpreterGuard *owned_guard; + } ensure; }; /* other API */ @@ -230,6 +275,21 @@ PyAPI_FUNC(void) PyThreadState_EnterTracing(PyThreadState *tstate); // function is set, otherwise disable them. PyAPI_FUNC(void) PyThreadState_LeaveTracing(PyThreadState *tstate); +#ifdef Py_STATS +#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +extern _Py_thread_local PyThreadState *_Py_tss_tstate; + +static inline PyStats* +_PyThreadState_GetStatsFast(void) +{ + if (_Py_tss_tstate == NULL) { + return NULL; // no attached thread state + } + return _Py_tss_tstate->pystats; +} +#endif +#endif // Py_STATS + /* PyGILState */ /* Helper/diagnostic function - return 1 if the current thread @@ -243,6 +303,18 @@ PyAPI_FUNC(int) PyGILState_Check(void); */ PyAPI_FUNC(PyObject*) _PyThread_CurrentFrames(void); +// Set the stack protection start address and stack protection size +// of a Python thread state +PyAPI_FUNC(int) PyUnstable_ThreadState_SetStackProtection( + PyThreadState *tstate, + void *stack_start_addr, // Stack start address + size_t stack_size); // Stack size (in bytes) + +// Reset the stack protection start address and stack protection size +// of a Python thread state +PyAPI_FUNC(void) PyUnstable_ThreadState_ResetStackProtection( + PyThreadState *tstate); + /* Routines for advanced debuggers, requested by David Beazley. Don't use unless you know what you are doing! */ PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Main(void); @@ -261,3 +333,8 @@ PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc( PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc( PyInterpreterState *interp, _PyFrameEvalFunction eval_frame); +PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameAllowSpecialization( + PyInterpreterState *interp, + int allow_specialization); +PyAPI_FUNC(int) _PyInterpreterState_IsSpecializationEnabled( + PyInterpreterState *interp); diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index cf830b6066f4aba..69659c48a3bf882 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -4,7 +4,7 @@ // // - _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF() // and Py_DECREF(). -// - _Py_stats variable +// - _PyStats_GET() // // Functions of the sys module: // @@ -14,7 +14,7 @@ // - sys._stats_dump() // // Python must be built with ./configure --enable-pystats to define the -// Py_STATS macro. +// _PyStats_GET() macro. // // Define _PY_INTERPRETER macro to increment interpreter_increfs and // interpreter_decrefs. Otherwise, increment increfs and decrefs. @@ -29,7 +29,7 @@ # error "this header file must not be included directly" #endif -#define PYSTATS_MAX_UOP_ID 1024 +#define PYSTATS_MAX_UOP_ID 2000 #define SPECIALIZATION_FAILURE_KINDS 60 @@ -109,6 +109,18 @@ typedef struct _gc_stats { uint64_t objects_not_transitively_reachable; } GCStats; +#ifdef Py_GIL_DISABLED +// stats specific to free-threaded build +typedef struct _ft_stats { + // number of times interpreter had to spin or park when trying to acquire a mutex + uint64_t mutex_sleeps; + // number of times that the QSBR mechanism polled (compute read sequence value) + uint64_t qsbr_polls; + // number of times stop-the-world mechanism was used + uint64_t world_stops; +} FTStats; +#endif + typedef struct _uop_stats { uint64_t execution_count; uint64_t miss; @@ -130,7 +142,9 @@ typedef struct _optimization_stats { uint64_t recursive_call; uint64_t low_confidence; uint64_t unknown_callee; + uint64_t trace_immediately_deopts; uint64_t executors_invalidated; + uint64_t fitness_terminated_traces; UOpStats opcode[PYSTATS_MAX_UOP_ID + 1]; uint64_t unsupported_opcode[256]; uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; @@ -138,6 +152,8 @@ typedef struct _optimization_stats { uint64_t optimized_trace_length_hist[_Py_UOP_HIST_SIZE]; uint64_t optimizer_attempts; uint64_t optimizer_successes; + uint64_t optimizer_contradiction; + uint64_t optimizer_frame_overflow; uint64_t optimizer_failure_reason_no_memory; uint64_t remove_globals_builtins_changed; uint64_t remove_globals_incorrect_keys; @@ -147,6 +163,7 @@ typedef struct _optimization_stats { uint64_t jit_code_size; uint64_t jit_trampoline_size; uint64_t jit_data_size; + uint64_t jit_got_size; uint64_t jit_padding_size; uint64_t jit_freed_memory_size; uint64_t trace_total_memory_hist[_Py_UOP_HIST_SIZE]; @@ -173,22 +190,48 @@ typedef struct _stats { CallStats call_stats; ObjectStats object_stats; OptimizationStats optimization_stats; +#ifdef Py_GIL_DISABLED + FTStats ft_stats; +#endif RareEventStats rare_event_stats; - GCStats *gc_stats; + GCStats gc_stats[3]; // must match NUM_GENERATIONS } PyStats; +// Export for most shared extensions +PyAPI_FUNC(PyStats *) _PyStats_GetLocal(void); + +#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +// use inline function version defined in cpython/pystate.h +static inline PyStats *_PyThreadState_GetStatsFast(void); +#define _PyStats_GET _PyThreadState_GetStatsFast +#else +#define _PyStats_GET _PyStats_GetLocal +#endif -// Export for shared extensions like 'math' -PyAPI_DATA(PyStats*) _Py_stats; +#define _Py_STATS_EXPR(expr) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (s != NULL) { \ + s->expr; \ + } \ + } while (0) + +#define _Py_STATS_COND_EXPR(cond, expr) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (s != NULL && (cond)) { \ + s->expr; \ + } \ + } while (0) #ifdef _PY_INTERPRETER -# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_increfs++; } while (0) -# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_decrefs++; } while (0) -# define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_increfs++; } while (0) -# define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_decrefs++; } while (0) +# define _Py_INCREF_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_increfs++) +# define _Py_DECREF_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_decrefs++) +# define _Py_INCREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_immortal_increfs++) +# define _Py_DECREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_immortal_decrefs++) #else -# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0) -# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0) -# define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_increfs++; } while (0) -# define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_decrefs++; } while (0) +# define _Py_INCREF_STAT_INC() _Py_STATS_EXPR(object_stats.increfs++) +# define _Py_DECREF_STAT_INC() _Py_STATS_EXPR(object_stats.decrefs++) +# define _Py_INCREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.immortal_increfs++) +# define _Py_DECREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.immortal_decrefs++) #endif diff --git a/Include/cpython/sentinelobject.h b/Include/cpython/sentinelobject.h new file mode 100644 index 000000000000000..e621d6abbfed8aa --- /dev/null +++ b/Include/cpython/sentinelobject.h @@ -0,0 +1,26 @@ +/* Sentinel object interface */ + +#ifndef Py_LIMITED_API +#ifndef _Py_SENTINELOBJECT_H +#define _Py_SENTINELOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +PyAPI_DATA(PyTypeObject) PySentinel_Type; + +#define PySentinel_CheckExact(op) Py_IS_TYPE((op), &PySentinel_Type) + +/* Alias as long as subclasses are not allowed. */ +#define PySentinel_Check(op) PySentinel_CheckExact(op) + +PyAPI_FUNC(PyObject *) PySentinel_New( + const char *name, + const char *module_name, + const char *repr); + +#ifdef __cplusplus +} +#endif +#endif /* !_Py_SENTINELOBJECT_H */ +#endif /* !Py_LIMITED_API */ diff --git a/Include/cpython/sliceobject.h b/Include/cpython/sliceobject.h new file mode 100644 index 000000000000000..137206eff15b33f --- /dev/null +++ b/Include/cpython/sliceobject.h @@ -0,0 +1,20 @@ +#ifndef _Py_CPYTHON_SLICEOBJECT_H +# error "this header file must not be included directly" +#endif + +/* Slice object interface */ + +/* +A slice object containing start, stop, and step data members (the +names are from range). After much talk with Guido, it was decided to +let these be any arbitrary python type. Py_None stands for omitted values. +*/ +typedef struct { + PyObject_HEAD + PyObject *start, *stop, *step; /* not NULL */ +} PySliceObject; + +PyAPI_FUNC(PyObject *) _PySlice_FromIndices(Py_ssize_t start, Py_ssize_t stop); +PyAPI_FUNC(int) _PySlice_GetLongIndices(PySliceObject *self, PyObject *length, + PyObject **start_ptr, PyObject **stop_ptr, + PyObject **step_ptr); diff --git a/Include/cpython/structseq.h b/Include/cpython/structseq.h new file mode 100644 index 000000000000000..83a1abcd6f3b347 --- /dev/null +++ b/Include/cpython/structseq.h @@ -0,0 +1,12 @@ +#ifndef _Py_CPYTHON_STRUCTSEQ_H +# error "this header file must not be included directly" +#endif + +PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type, + PyStructSequence_Desc *desc); +PyAPI_FUNC(int) PyStructSequence_InitType2(PyTypeObject *type, + PyStructSequence_Desc *desc); + +typedef PyTupleObject PyStructSequence; +#define PyStructSequence_SET_ITEM PyStructSequence_SetItem +#define PyStructSequence_GET_ITEM PyStructSequence_GetItem diff --git a/Include/cpython/traceback.h b/Include/cpython/traceback.h index 81c51944f136f29..7f42730f1b09194 100644 --- a/Include/cpython/traceback.h +++ b/Include/cpython/traceback.h @@ -11,3 +11,11 @@ struct _traceback { int tb_lasti; int tb_lineno; }; + +PyAPI_FUNC(const char*) PyUnstable_DumpTraceback(int fd, PyThreadState *tstate); + +PyAPI_FUNC(const char*) PyUnstable_DumpTracebackThreads( + int fd, + PyInterpreterState *interp, + PyThreadState *current_tstate, + Py_ssize_t max_threads); diff --git a/Include/cpython/tupleobject.h b/Include/cpython/tupleobject.h index afb98ccbb81b2de..888baaf3358267e 100644 --- a/Include/cpython/tupleobject.h +++ b/Include/cpython/tupleobject.h @@ -38,3 +38,7 @@ PyTuple_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) { } #define PyTuple_SET_ITEM(op, index, value) \ PyTuple_SET_ITEM(_PyObject_CAST(op), (index), _PyObject_CAST(value)) + +PyAPI_FUNC(PyObject*) PyTuple_FromArray( + PyObject *const *array, + Py_ssize_t size); diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 73e3bc44d6c9ca6..ea91f4158eb3929 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -301,7 +301,6 @@ static inline Py_ssize_t PyUnicode_GET_LENGTH(PyObject *op) { /* Returns the cached hash, or -1 if not cached yet. */ static inline Py_hash_t PyUnstable_Unicode_GET_CACHED_HASH(PyObject *op) { - assert(PyUnicode_Check(op)); #ifdef Py_GIL_DISABLED return _Py_atomic_load_ssize_relaxed(&_PyASCIIObject_CAST(op)->hash); #else @@ -497,7 +496,7 @@ PyAPI_FUNC(int) PyUnicodeWriter_WriteWideChar( Py_ssize_t size); PyAPI_FUNC(int) PyUnicodeWriter_WriteUCS4( PyUnicodeWriter *writer, - Py_UCS4 *str, + const Py_UCS4 *str, Py_ssize_t size); PyAPI_FUNC(int) PyUnicodeWriter_WriteStr( @@ -779,4 +778,4 @@ static inline int Py_UNICODE_ISALNUM(Py_UCS4 ch) { // Return an interned Unicode object for an Identifier; may fail if there is no // memory. -PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*); +Py_DEPRECATED(3.15) PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*); diff --git a/Include/cpython/warnings.h b/Include/cpython/warnings.h index 8731fd2e96b716f..4e3eb88e8ff4472 100644 --- a/Include/cpython/warnings.h +++ b/Include/cpython/warnings.h @@ -18,9 +18,3 @@ PyAPI_FUNC(int) PyErr_WarnExplicitFormat( // DEPRECATED: Use PyErr_WarnEx() instead. #define PyErr_Warn(category, msg) PyErr_WarnEx((category), (msg), 1) - -int _PyErr_WarnExplicitObjectWithContext( - PyObject *category, - PyObject *message, - PyObject *filename, - int lineno); diff --git a/Include/cpython/weakrefobject.h b/Include/cpython/weakrefobject.h index da8e77cddaca638..e0711407cee4708 100644 --- a/Include/cpython/weakrefobject.h +++ b/Include/cpython/weakrefobject.h @@ -47,20 +47,3 @@ PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); // Test if a weak reference is dead. PyAPI_FUNC(int) PyWeakref_IsDead(PyObject *ref); - -Py_DEPRECATED(3.13) static inline PyObject* PyWeakref_GET_OBJECT(PyObject *ref_obj) -{ - PyWeakReference *ref = _PyWeakref_CAST(ref_obj); - PyObject *obj = ref->wr_object; - // Explanation for the Py_REFCNT() check: when a weakref's target is part - // of a long chain of deallocations which triggers the trashcan mechanism, - // clearing the weakrefs can be delayed long after the target's refcount - // has dropped to zero. In the meantime, code accessing the weakref will - // be able to "see" the target object even though it is supposed to be - // unreachable. See issue gh-60806. - if (Py_REFCNT(obj) > 0) { - return obj; - } - return Py_None; -} -#define PyWeakref_GET_OBJECT(ref) PyWeakref_GET_OBJECT(_PyObject_CAST(ref)) diff --git a/Include/critical_section.h b/Include/critical_section.h index 3b37615a8b17e2a..732bfab7ecf2349 100644 --- a/Include/critical_section.h +++ b/Include/critical_section.h @@ -4,6 +4,91 @@ extern "C" { #endif +// Python critical sections +// +// Conceptually, critical sections are a deadlock avoidance layer on top of +// per-object locks. These helpers, in combination with those locks, replace +// our usage of the global interpreter lock to provide thread-safety for +// otherwise thread-unsafe objects, such as dict. +// +// NOTE: These APIs are no-ops in non-free-threaded builds. +// +// NOTE: Only the top-most critical section is guaranteed to be active. +// Operations that need to lock two objects at once must use +// `Py_BEGIN_CRITICAL_SECTION2()`. You *CANNOT* use nested critical sections +// to lock more than one object at once, because the inner critical section +// may suspend the outer critical sections. This API does not provide a way +// to lock more than two objects at once (though it could be added later +// if actually needed). +// +// NOTE: Critical sections implicitly behave like reentrant locks because +// attempting to acquire the same lock will suspend any outer (earlier) +// critical sections. However, they are less efficient for this use case than +// purposefully designed reentrant locks. +// +// Example usage: +// Py_BEGIN_CRITICAL_SECTION(op); +// ... +// Py_END_CRITICAL_SECTION(); +// +// To lock two objects at once: +// Py_BEGIN_CRITICAL_SECTION2(op1, op2); +// ... +// Py_END_CRITICAL_SECTION2(); + +// NOTE: the contents of this struct are private and their meaning may +// change betweeen Python releases without a deprecation period. +typedef struct PyCriticalSection { + // Tagged pointer to an outer active critical section (or 0). + uintptr_t _cs_prev; + + // Mutex used to protect critical section + struct PyMutex *_cs_mutex; +} PyCriticalSection; + +// A critical section protected by two mutexes. Use +// Py_BEGIN_CRITICAL_SECTION2 and Py_END_CRITICAL_SECTION2. +// NOTE: the contents of this struct are private and may change betweeen +// Python releases without a deprecation period. +typedef struct PyCriticalSection2 { + PyCriticalSection _cs_base; + + struct PyMutex *_cs_mutex2; +} PyCriticalSection2; + +PyAPI_FUNC(void) +PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op); + +PyAPI_FUNC(void) +PyCriticalSection_End(PyCriticalSection *c); + +PyAPI_FUNC(void) +PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b); + +PyAPI_FUNC(void) +PyCriticalSection2_End(PyCriticalSection2 *c); + +// These are definitions for the stable ABI. For GIL-ful builds they're +// conditionally redefined as no-ops in cpython/critical_section.h. + +# define Py_BEGIN_CRITICAL_SECTION(op) \ + { \ + PyCriticalSection _py_cs; \ + PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op)) + +# define Py_END_CRITICAL_SECTION() \ + PyCriticalSection_End(&_py_cs); \ + } + +# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyCriticalSection2_Begin(&_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) + +# define Py_END_CRITICAL_SECTION2() \ + PyCriticalSection2_End(&_py_cs2); \ + } + #ifndef Py_LIMITED_API # define Py_CPYTHON_CRITICAL_SECTION_H # include "cpython/critical_section.h" diff --git a/Include/datetime.h b/Include/datetime.h index b78cc0e8e2e5acc..66e6c6e3ac35755 100644 --- a/Include/datetime.h +++ b/Include/datetime.h @@ -1,8 +1,8 @@ /* datetime.h */ #ifndef Py_LIMITED_API -#ifndef DATETIME_H -#define DATETIME_H +#ifndef Py_DATETIME_H +#define Py_DATETIME_H #ifdef __cplusplus extern "C" { #endif @@ -196,8 +196,23 @@ typedef struct { /* Define global variable for the C API and a macro for setting it. */ static PyDateTime_CAPI *PyDateTimeAPI = NULL; -#define PyDateTime_IMPORT \ - PyDateTimeAPI = (PyDateTime_CAPI *)PyCapsule_Import(PyDateTime_CAPSULE_NAME, 0) +static inline PyDateTime_CAPI * +_PyDateTime_IMPORT(void) { + PyDateTime_CAPI *val = (PyDateTime_CAPI *)_Py_atomic_load_ptr(&PyDateTimeAPI); + if (val == NULL) { + PyDateTime_CAPI *capi = (PyDateTime_CAPI *)PyCapsule_Import( + PyDateTime_CAPSULE_NAME, 0); + if (capi != NULL) { + /* if the compare exchange fails then in that case + another thread would have initialized it */ + _Py_atomic_compare_exchange_ptr(&PyDateTimeAPI, &val, (void *)capi); + return capi; + } + } + return val; +} + +#define PyDateTime_IMPORT _PyDateTime_IMPORT() /* Macro for access to the UTC singleton */ #define PyDateTime_TimeZone_UTC PyDateTimeAPI->TimeZone_UTC @@ -263,5 +278,5 @@ static PyDateTime_CAPI *PyDateTimeAPI = NULL; #ifdef __cplusplus } #endif -#endif +#endif /* !Py_DATETIME_H */ #endif /* !Py_LIMITED_API */ diff --git a/Include/descrobject.h b/Include/descrobject.h index fd66d17b497a31f..340de4e0e1e6ffc 100644 --- a/Include/descrobject.h +++ b/Include/descrobject.h @@ -80,10 +80,14 @@ struct PyMemberDef { #define _Py_T_NONE 20 // Deprecated. Value is always None. /* Flags */ -#define Py_READONLY 1 -#define Py_AUDIT_READ 2 // Added in 3.10, harmless no-op before that -#define _Py_WRITE_RESTRICTED 4 // Deprecated, no-op. Do not reuse the value. -#define Py_RELATIVE_OFFSET 8 +#define Py_READONLY (1 << 0) +#define Py_AUDIT_READ (1 << 1) // Added in 3.10, harmless no-op before that +#define _Py_WRITE_RESTRICTED (1 << 2) // Deprecated, no-op. Do not reuse the value. +#define Py_RELATIVE_OFFSET (1 << 3) + +#ifndef Py_LIMITED_API +# define _Py_AFTER_ITEMS (1 << 4) // For internal use. +#endif PyAPI_FUNC(PyObject *) PyMember_GetOne(const char *, PyMemberDef *); PyAPI_FUNC(int) PyMember_SetOne(char *, PyMemberDef *, PyObject *); diff --git a/Include/dictobject.h b/Include/dictobject.h index 1bbeec1ab699e75..0384e3131dcdb52 100644 --- a/Include/dictobject.h +++ b/Include/dictobject.h @@ -68,6 +68,18 @@ PyAPI_FUNC(int) PyDict_GetItemRef(PyObject *mp, PyObject *key, PyObject **result PyAPI_FUNC(int) PyDict_GetItemStringRef(PyObject *mp, const char *key, PyObject **result); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030F0000 +// Inserts `key` with a value `default_value`, if `key` is not already present +// in the dictionary. If `result` is not NULL, then the value associated +// with `key` is returned in `*result` (either the existing value, or the now +// inserted `default_value`). +// Returns: +// -1 on error +// 0 if `key` was not present and `default_value` was inserted +// 1 if `key` was present and `default_value` was not inserted +PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); +#endif + #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 PyAPI_FUNC(PyObject *) PyObject_GenericGetDict(PyObject *, void *); #endif diff --git a/Include/dynamic_annotations.h b/Include/dynamic_annotations.h index 4d4def9bf8983e2..5290319dd762c55 100644 --- a/Include/dynamic_annotations.h +++ b/Include/dynamic_annotations.h @@ -461,6 +461,7 @@ int RunningOnValgrind(void); #if DYNAMIC_ANNOTATIONS_ENABLED != 0 && defined(__cplusplus) +extern "C++" { /* _Py_ANNOTATE_UNPROTECTED_READ is the preferred way to annotate racey reads. Instead of doing @@ -476,6 +477,8 @@ int RunningOnValgrind(void); _Py_ANNOTATE_IGNORE_READS_END(); return res; } +} + /* Apply _Py_ANNOTATE_BENIGN_RACE_SIZED to a static variable. */ #define _Py_ANNOTATE_BENIGN_RACE_STATIC(static_var, description) \ namespace { \ diff --git a/Include/exports.h b/Include/exports.h index 0c646d5beb6ad6b..18692283005e596 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -9,6 +9,7 @@ inside the Python core, they are private to the core. If in an extension module, it may be declared with external linkage depending on the platform. + PyMODEXPORT_FUNC: Like PyMODINIT_FUNC, but for a slots array As a number of platforms support/require "__declspec(dllimport/dllexport)", we support a HAVE_DECLSPEC_DLL macro to save duplication. @@ -34,6 +35,12 @@ #define Py_EXPORTED_SYMBOL #define Py_LOCAL_SYMBOL #endif + /* module init functions outside the core must be exported */ + #if defined(_PyEXPORTS_CORE) + #define _PyINIT_EXPORTED_SYMBOL Py_EXPORTED_SYMBOL + #else + #define _PyINIT_EXPORTED_SYMBOL __declspec(dllexport) + #endif #else /* * If we only ever used gcc >= 5, we could use __has_attribute(visibility) @@ -51,22 +58,19 @@ #define Py_EXPORTED_SYMBOL #define Py_LOCAL_SYMBOL #endif + #define _PyINIT_EXPORTED_SYMBOL Py_EXPORTED_SYMBOL #endif /* only get special linkage if built as shared or platform is Cygwin */ #if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__) # if defined(HAVE_DECLSPEC_DLL) -# if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE -# define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE +# if defined(_PyEXPORTS_CORE) && !defined(_PyEXPORTS_CORE_MODULE) /* module init functions inside the core need no external linkage */ /* except for Cygwin to handle embedding */ -# if defined(__CYGWIN__) -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* -# else /* __CYGWIN__ */ -# define PyMODINIT_FUNC PyObject* +# if !defined(__CYGWIN__) +# define _PyINIT_FUNC_DECLSPEC # endif /* __CYGWIN__ */ -# else /* Py_BUILD_CORE */ +# else /* _PyEXPORTS_CORE */ /* Building an extension module, or an embedded situation */ /* public Python functions and data are imported */ /* Under Cygwin, auto-import functions to prevent compilation */ @@ -76,13 +80,7 @@ # define PyAPI_FUNC(RTYPE) Py_IMPORTED_SYMBOL RTYPE # endif /* !__CYGWIN__ */ # define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE - /* module init functions outside the core must be exported */ -# if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* -# else /* __cplusplus */ -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* -# endif /* __cplusplus */ -# endif /* Py_BUILD_CORE */ +# endif /* _PyEXPORTS_CORE */ # endif /* HAVE_DECLSPEC_DLL */ #endif /* Py_ENABLE_SHARED */ @@ -93,13 +91,19 @@ #ifndef PyAPI_DATA # define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE #endif -#ifndef PyMODINIT_FUNC +#ifndef _PyINIT_FUNC_DECLSPEC # if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC extern "C" _PyINIT_EXPORTED_SYMBOL # else /* __cplusplus */ -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC _PyINIT_EXPORTED_SYMBOL # endif /* __cplusplus */ #endif +#ifndef PyMODINIT_FUNC + #define PyMODINIT_FUNC _PyINIT_FUNC_DECLSPEC PyObject* +#endif +#ifndef PyMODEXPORT_FUNC + #define PyMODEXPORT_FUNC _PyINIT_FUNC_DECLSPEC PySlot* +#endif #endif /* Py_EXPORTS_H */ diff --git a/Include/floatobject.h b/Include/floatobject.h index 4d24a76edd5de1b..814337b070ab50a 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -18,14 +18,14 @@ PyAPI_DATA(PyTypeObject) PyFloat_Type; #define Py_RETURN_NAN return PyFloat_FromDouble(Py_NAN) -#define Py_RETURN_INF(sign) \ - do { \ - if (copysign(1., sign) == 1.) { \ - return PyFloat_FromDouble(Py_INFINITY); \ - } \ - else { \ - return PyFloat_FromDouble(-Py_INFINITY); \ - } \ +#define Py_RETURN_INF(sign) \ + do { \ + if (copysign(1., sign) == 1.) { \ + return PyFloat_FromDouble(INFINITY); \ + } \ + else { \ + return PyFloat_FromDouble(-INFINITY); \ + } \ } while(0) PyAPI_FUNC(double) PyFloat_GetMax(void); diff --git a/Include/import.h b/Include/import.h index d91ebe96ca868d1..c062e46bff46bf7 100644 --- a/Include/import.h +++ b/Include/import.h @@ -88,6 +88,19 @@ PyAPI_FUNC(int) PyImport_AppendInittab( PyObject* (*initfunc)(void) ); +typedef enum { + PyImport_LAZY_NORMAL, + PyImport_LAZY_ALL +} PyImport_LazyImportsMode; + +#ifndef Py_LIMITED_API +PyAPI_FUNC(int) PyImport_SetLazyImportsMode(PyImport_LazyImportsMode mode); +PyAPI_FUNC(int) PyImport_SetLazyImportsFilter(PyObject *filter); + +PyAPI_FUNC(PyImport_LazyImportsMode) PyImport_GetLazyImportsMode(void); +PyAPI_FUNC(PyObject *) PyImport_GetLazyImportsFilter(void); +#endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_IMPORT_H # include "cpython/import.h" diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h index 19e932241743141..286e7bf668312db 100644 --- a/Include/internal/mimalloc/mimalloc/types.h +++ b/Include/internal/mimalloc/mimalloc/types.h @@ -608,8 +608,8 @@ struct mi_heap_s { #if (MI_DEBUG) // use our own assertion to print without memory allocation -mi_decl_noreturn mi_decl_cold mi_decl_throw -void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func); +mi_decl_noreturn mi_decl_cold +void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func) mi_decl_throw; #define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) #else #define mi_assert(x) diff --git a/Include/internal/pycore_abstract.h b/Include/internal/pycore_abstract.h index 3cc0afac4bd5b45..67c6fa7c0c4ed5b 100644 --- a/Include/internal/pycore_abstract.h +++ b/Include/internal/pycore_abstract.h @@ -16,10 +16,11 @@ _PyIndex_Check(PyObject *obj) return (tp_as_number != NULL && tp_as_number->nb_index != NULL); } -PyObject *_PyNumber_PowerNoMod(PyObject *lhs, PyObject *rhs); -PyObject *_PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs); +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyNumber_PowerNoMod(PyObject *lhs, PyObject *rhs); +PyAPI_FUNC(PyObject *) _PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs); -extern int _PyObject_HasLen(PyObject *o); +PyAPI_FUNC(int) _PyObject_HasLen(PyObject *o); /* === Sequence protocol ================================================ */ @@ -51,10 +52,22 @@ extern int _PyObject_RealIsSubclass(PyObject *derived, PyObject *cls); // Export for '_bisect' shared extension. PyAPI_FUNC(int) _Py_convert_optional_to_ssize_t(PyObject *, void *); +// Convert Python int to Py_ssize_t. Do nothing if the argument is None. +// Raises ValueError if argument is negative. +PyAPI_FUNC(int) _Py_convert_optional_to_non_negative_ssize_t(PyObject *, void *); + // Same as PyNumber_Index() but can return an instance of a subclass of int. // Export for 'math' shared extension. PyAPI_FUNC(PyObject*) _PyNumber_Index(PyObject *o); +typedef struct { + PyObject *object; + PySendResult kind; +} PySendResultPair; + +// Same as PyIter_Send but returns a struct for MSVC tailcall support +PyAPI_FUNC(PySendResultPair) _PyIter_Send(PyObject *iter, PyObject *arg); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index 60367202bab6370..b47398669bbe513 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -329,12 +329,14 @@ struct _stmt { struct { asdl_alias_seq *names; + int is_lazy; } Import; struct { identifier module; asdl_alias_seq *names; int level; + int is_lazy; } ImportFrom; struct { @@ -764,11 +766,12 @@ stmt_ty _PyAST_TryStar(asdl_stmt_seq * body, asdl_excepthandler_seq * handlers, end_col_offset, PyArena *arena); stmt_ty _PyAST_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -stmt_ty _PyAST_Import(asdl_alias_seq * names, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); +stmt_ty _PyAST_Import(asdl_alias_seq * names, int is_lazy, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); stmt_ty _PyAST_ImportFrom(identifier module, asdl_alias_seq * names, int level, - int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); + int is_lazy, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); stmt_ty _PyAST_Global(asdl_identifier_seq * names, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); stmt_ty _PyAST_Nonlocal(asdl_identifier_seq * names, int lineno, int diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index d4ac419f51d6b2d..32c12fb5875e8ea 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -161,6 +161,7 @@ struct ast_state { PyObject *__module__; PyObject *_attributes; PyObject *_fields; + PyObject *abstract_types; PyObject *alias_type; PyObject *annotation; PyObject *arg; @@ -205,6 +206,7 @@ struct ast_state { PyObject *id; PyObject *ifs; PyObject *is_async; + PyObject *is_lazy; PyObject *items; PyObject *iter; PyObject *key; diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 454c8dde031ff42..38dd82f6fc8a140 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -12,6 +12,7 @@ extern "C" { #include #include #include "pycore_structs.h" // _Py_BackoffCounter +#include "pycore_interp_structs.h" // _PyOptimizationConfig /* 16-bit countdown counters using exponential backoff. @@ -22,33 +23,48 @@ extern "C" { Another use is for the Tier 2 optimizer to decide when to create a new Tier 2 trace (executor). Again, exponential backoff is used. - The 16-bit counter is structured as a 12-bit unsigned 'value' - and a 4-bit 'backoff' field. When resetting the counter, the + The 16-bit counter is structured as a 13-bit unsigned 'value' + and a 3-bit 'backoff' field. When resetting the counter, the backoff field is incremented (until it reaches a limit) and the - value is set to a bit mask representing the value 2**backoff - 1. - The maximum backoff is 12 (the number of bits in the value). + value is set to a bit mask representing some prime value - 1. + New values and backoffs for each backoff are calculated once + at compile time and saved to value_and_backoff_next table. + The maximum backoff is 6, since 7 is an UNREACHABLE_BACKOFF. There is an exceptional value which must not be updated, 0xFFFF. */ -#define BACKOFF_BITS 4 -#define MAX_BACKOFF 12 -#define UNREACHABLE_BACKOFF 15 - -static inline bool -is_unreachable_backoff_counter(_Py_BackoffCounter counter) -{ - return counter.value_and_backoff == UNREACHABLE_BACKOFF; -} +#define BACKOFF_BITS 3 +#define BACKOFF_MASK 7 +#define MAX_BACKOFF 6 +#define UNREACHABLE_BACKOFF 7 +#define MAX_VALUE 0x1FFF + +#define MAKE_VALUE_AND_BACKOFF(value, backoff) \ + ((uint16_t)(((value) << BACKOFF_BITS) | (backoff))) + +// For previous backoff b we use value x such that +// x + 1 is near to 2**(2*b+1) and x + 1 is prime. +static const uint16_t value_and_backoff_next[] = { + MAKE_VALUE_AND_BACKOFF(1, 1), + MAKE_VALUE_AND_BACKOFF(6, 2), + MAKE_VALUE_AND_BACKOFF(30, 3), + MAKE_VALUE_AND_BACKOFF(126, 4), + MAKE_VALUE_AND_BACKOFF(508, 5), + MAKE_VALUE_AND_BACKOFF(2052, 6), + // We use the same backoff counter for all backoffs >= MAX_BACKOFF. + MAKE_VALUE_AND_BACKOFF(8190, 6), + MAKE_VALUE_AND_BACKOFF(8190, 6), +}; static inline _Py_BackoffCounter make_backoff_counter(uint16_t value, uint16_t backoff) { - assert(backoff <= 15); - assert(value <= 0xFFF); - _Py_BackoffCounter result; - result.value_and_backoff = (value << BACKOFF_BITS) | backoff; - return result; + assert(backoff <= UNREACHABLE_BACKOFF); + assert(value <= MAX_VALUE); + return ((_Py_BackoffCounter){ + .value_and_backoff = MAKE_VALUE_AND_BACKOFF(value, backoff) + }); } static inline _Py_BackoffCounter @@ -62,14 +78,11 @@ forge_backoff_counter(uint16_t counter) static inline _Py_BackoffCounter restart_backoff_counter(_Py_BackoffCounter counter) { - assert(!is_unreachable_backoff_counter(counter)); - int backoff = counter.value_and_backoff & 15; - if (backoff < MAX_BACKOFF) { - return make_backoff_counter((1 << (backoff + 1)) - 1, backoff + 1); - } - else { - return make_backoff_counter((1 << MAX_BACKOFF) - 1, MAX_BACKOFF); - } + uint16_t backoff = counter.value_and_backoff & BACKOFF_MASK; + assert(backoff <= MAX_BACKOFF); + return ((_Py_BackoffCounter){ + .value_and_backoff = value_and_backoff_next[backoff] + }); } static inline _Py_BackoffCounter @@ -95,31 +108,60 @@ backoff_counter_triggers(_Py_BackoffCounter counter) return counter.value_and_backoff < UNREACHABLE_BACKOFF; } +static inline _Py_BackoffCounter +trigger_backoff_counter(void) +{ + _Py_BackoffCounter result; + result.value_and_backoff = 0; + return result; +} + // Initial JUMP_BACKWARD counter. // Must be larger than ADAPTIVE_COOLDOWN_VALUE, otherwise when JIT code is // invalidated we may construct a new trace before the bytecode has properly // re-specialized: -#define JUMP_BACKWARD_INITIAL_VALUE 4095 -#define JUMP_BACKWARD_INITIAL_BACKOFF 12 +// Note: this should be a prime number-1. This increases the likelihood of +// finding a "good" loop iteration to trace. +// For example, 4095 does not work for the nqueens benchmark on pyperformance +// as we always end up tracing the loop iteration's +// exhaustion iteration. Which aborts our current tracer. +#define JUMP_BACKWARD_INITIAL_VALUE 4000 +#define JUMP_BACKWARD_INITIAL_BACKOFF 6 +static inline _Py_BackoffCounter +initial_jump_backoff_counter(_PyOptimizationConfig *opt_config) +{ + return make_backoff_counter( + opt_config->jump_backward_initial_value, + opt_config->jump_backward_initial_backoff); +} + +// This needs to be around 2-4x of JUMP_BACKWARD_INITIAL_VALUE +// The reasoning is that we always want loop traces to form and inline +// functions before functions themselves warm up and link to them instead +// of inlining. +#define RESUME_INITIAL_VALUE 8190 +#define RESUME_INITIAL_BACKOFF 6 static inline _Py_BackoffCounter -initial_jump_backoff_counter(void) +initial_resume_backoff_counter(_PyOptimizationConfig *opt_config) { - return make_backoff_counter(JUMP_BACKWARD_INITIAL_VALUE, - JUMP_BACKWARD_INITIAL_BACKOFF); + return make_backoff_counter( + opt_config->resume_initial_value, + opt_config->resume_initial_backoff); } /* Initial exit temperature. * Must be larger than ADAPTIVE_COOLDOWN_VALUE, * otherwise when a side exit warms up we may construct * a new trace before the Tier 1 code has properly re-specialized. */ -#define SIDE_EXIT_INITIAL_VALUE 4095 -#define SIDE_EXIT_INITIAL_BACKOFF 12 +#define SIDE_EXIT_INITIAL_VALUE 4000 +#define SIDE_EXIT_INITIAL_BACKOFF 6 static inline _Py_BackoffCounter -initial_temperature_backoff_counter(void) +initial_temperature_backoff_counter(_PyOptimizationConfig *opt_config) { - return make_backoff_counter(SIDE_EXIT_INITIAL_VALUE, - SIDE_EXIT_INITIAL_BACKOFF); + return make_backoff_counter( + opt_config->side_exit_initial_value, + opt_config->side_exit_initial_backoff); } /* Unreachable backoff counter. */ diff --git a/Include/internal/pycore_blocks_output_buffer.h b/Include/internal/pycore_blocks_output_buffer.h index 573e10359b7bd27..322c1e93344ba30 100644 --- a/Include/internal/pycore_blocks_output_buffer.h +++ b/Include/internal/pycore_blocks_output_buffer.h @@ -45,12 +45,14 @@ extern "C" { #endif typedef struct { - // List of bytes objects - PyObject *list; + // Bytes writer managing output buffer + PyBytesWriter *writer; // Number of whole allocated size Py_ssize_t allocated; - // Max length of the buffer, negative number means unlimited length. + // Max length of the buffer, negative number means unlimited length Py_ssize_t max_length; + // Number of blocks of bytes. Used to calculate next allocation size + size_t num_blocks; } _BlocksOutputBuffer; static const char unable_allocate_msg[] = "Unable to allocate output buffer."; @@ -107,11 +109,10 @@ _BlocksOutputBuffer_InitAndGrow(_BlocksOutputBuffer *buffer, const Py_ssize_t max_length, void **next_out) { - PyObject *b; Py_ssize_t block_size; - // ensure .list was set to NULL - assert(buffer->list == NULL); + // ensure .writer was set to NULL + assert(buffer->writer == NULL); // get block size if (0 <= max_length && max_length < BUFFER_BLOCK_SIZE[0]) { @@ -120,25 +121,17 @@ _BlocksOutputBuffer_InitAndGrow(_BlocksOutputBuffer *buffer, block_size = BUFFER_BLOCK_SIZE[0]; } - // the first block - b = PyBytes_FromStringAndSize(NULL, block_size); - if (b == NULL) { + buffer->writer = PyBytesWriter_Create(block_size); + if (buffer->writer == NULL) { return -1; } - // create the list - buffer->list = PyList_New(1); - if (buffer->list == NULL) { - Py_DECREF(b); - return -1; - } - PyList_SET_ITEM(buffer->list, 0, b); - // set variables buffer->allocated = block_size; buffer->max_length = max_length; + buffer->num_blocks = 1; - *next_out = PyBytes_AS_STRING(b); + *next_out = PyBytesWriter_GetData(buffer->writer); return block_size; } @@ -155,31 +148,21 @@ _BlocksOutputBuffer_InitWithSize(_BlocksOutputBuffer *buffer, const Py_ssize_t init_size, void **next_out) { - PyObject *b; - // ensure .list was set to NULL - assert(buffer->list == NULL); + // ensure .writer was set to NULL + assert(buffer->writer == NULL); - // the first block - b = PyBytes_FromStringAndSize(NULL, init_size); - if (b == NULL) { - PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); + buffer->writer = PyBytesWriter_Create(init_size); + if (buffer->writer == NULL) { return -1; } - // create the list - buffer->list = PyList_New(1); - if (buffer->list == NULL) { - Py_DECREF(b); - return -1; - } - PyList_SET_ITEM(buffer->list, 0, b); - // set variables buffer->allocated = init_size; buffer->max_length = -1; + buffer->num_blocks = 1; - *next_out = PyBytes_AS_STRING(b); + *next_out = PyBytesWriter_GetData(buffer->writer); return init_size; } @@ -193,8 +176,6 @@ _BlocksOutputBuffer_Grow(_BlocksOutputBuffer *buffer, void **next_out, const Py_ssize_t avail_out) { - PyObject *b; - const Py_ssize_t list_len = Py_SIZE(buffer->list); Py_ssize_t block_size; // ensure no gaps in the data @@ -205,11 +186,10 @@ _BlocksOutputBuffer_Grow(_BlocksOutputBuffer *buffer, } // get block size - if (list_len < (Py_ssize_t) Py_ARRAY_LENGTH(BUFFER_BLOCK_SIZE)) { - block_size = BUFFER_BLOCK_SIZE[list_len]; - } else { - block_size = BUFFER_BLOCK_SIZE[Py_ARRAY_LENGTH(BUFFER_BLOCK_SIZE) - 1]; - } + size_t maxblock = Py_ARRAY_LENGTH(BUFFER_BLOCK_SIZE); + assert(maxblock >= 1); + size_t block_index = Py_MIN(buffer->num_blocks, maxblock - 1); + block_size = BUFFER_BLOCK_SIZE[block_index]; // check max_length if (buffer->max_length >= 0) { @@ -229,22 +209,19 @@ _BlocksOutputBuffer_Grow(_BlocksOutputBuffer *buffer, return -1; } - // create the block - b = PyBytes_FromStringAndSize(NULL, block_size); - if (b == NULL) { + if (PyBytesWriter_Grow(buffer->writer, block_size)) { PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); return -1; } - if (PyList_Append(buffer->list, b) < 0) { - Py_DECREF(b); - return -1; - } - Py_DECREF(b); + + Py_ssize_t current_size = buffer->allocated; // set variables buffer->allocated += block_size; + buffer->num_blocks += 1; - *next_out = PyBytes_AS_STRING(b); + char *data = PyBytesWriter_GetData(buffer->writer); + *next_out = data + current_size; return block_size; } @@ -265,54 +242,20 @@ static inline PyObject * _BlocksOutputBuffer_Finish(_BlocksOutputBuffer *buffer, const Py_ssize_t avail_out) { - PyObject *result, *block; - const Py_ssize_t list_len = Py_SIZE(buffer->list); - - // fast path for single block - if ((list_len == 1 && avail_out == 0) || - (list_len == 2 && Py_SIZE(PyList_GET_ITEM(buffer->list, 1)) == avail_out)) - { - block = PyList_GET_ITEM(buffer->list, 0); - Py_INCREF(block); - - Py_CLEAR(buffer->list); - return block; - } - - // final bytes object - result = PyBytes_FromStringAndSize(NULL, buffer->allocated - avail_out); - if (result == NULL) { - PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); - return NULL; - } - - // memory copy - if (list_len > 0) { - char *posi = PyBytes_AS_STRING(result); - - // blocks except the last one - Py_ssize_t i = 0; - for (; i < list_len-1; i++) { - block = PyList_GET_ITEM(buffer->list, i); - memcpy(posi, PyBytes_AS_STRING(block), Py_SIZE(block)); - posi += Py_SIZE(block); - } - // the last block - block = PyList_GET_ITEM(buffer->list, i); - memcpy(posi, PyBytes_AS_STRING(block), Py_SIZE(block) - avail_out); - } else { - assert(Py_SIZE(result) == 0); - } - - Py_CLEAR(buffer->list); - return result; + PyObject *obj; + assert(buffer->writer != NULL); + obj = PyBytesWriter_FinishWithSize(buffer->writer, + buffer->allocated - avail_out); + buffer->writer = NULL; + return obj; } /* Clean up the buffer when an error occurred. */ static inline void _BlocksOutputBuffer_OnError(_BlocksOutputBuffer *buffer) { - Py_CLEAR(buffer->list); + PyBytesWriter_Discard(buffer->writer); + buffer->writer = NULL; } #ifdef __cplusplus diff --git a/Include/internal/pycore_bytes_methods.h b/Include/internal/pycore_bytes_methods.h index 059dc2599bbd77e..3e1474c1c010f96 100644 --- a/Include/internal/pycore_bytes_methods.h +++ b/Include/internal/pycore_bytes_methods.h @@ -47,6 +47,9 @@ extern PyObject *_Py_bytes_endswith(const char *str, Py_ssize_t len, /* The maketrans() static method. */ extern PyObject* _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to); +/* Helper for repr(bytes) and repr(bytearray). */ +extern PyObject *_Py_bytes_repr(const char *, Py_ssize_t, int, const char *); + /* Shared __doc__ strings. */ extern const char _Py_isspace__doc__[]; extern const char _Py_isalpha__doc__[]; diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index 8ea9b3ebb884541..27a7a46152f57b8 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -14,6 +14,11 @@ extern PyObject* _PyBytes_FormatEx( PyObject *args, int use_bytearray); +/* Concatenate two bytes objects. Used as the sq_concat slot and by the + * specializing interpreter. Unlike PyBytes_Concat(), this returns a new + * reference rather than modifying its first argument in place. */ +extern PyObject* _PyBytes_Concat(PyObject *a, PyObject *b); + extern PyObject* _PyBytes_FromHex( PyObject *string, int use_bytearray); @@ -57,91 +62,51 @@ _PyBytes_ReverseFind(const char *haystack, Py_ssize_t len_haystack, // // Export for 'array' shared extension. PyAPI_FUNC(void) -_PyBytes_Repeat(char* dest, Py_ssize_t len_dest, +_PyBytes_RepeatBuffer(char* dest, Py_ssize_t len_dest, const char* src, Py_ssize_t len_src); -/* --- _PyBytesWriter ----------------------------------------------------- */ +PyAPI_FUNC(PyObject *) _PyBytes_Repeat(PyObject *self, Py_ssize_t n); -/* The _PyBytesWriter structure is big: it contains an embedded "stack buffer". - A _PyBytesWriter variable must be declared at the end of variables in a - function to optimize the memory allocation on the stack. */ -typedef struct { - /* bytes, bytearray or NULL (when the small buffer is used) */ - PyObject *buffer; +/* _PyBytesObject_SIZE gives the basic size of a bytes object; any memory allocation + for a bytes object of length n should request PyBytesObject_SIZE + n bytes. - /* Number of allocated size. */ - Py_ssize_t allocated; + Using _PyBytesObject_SIZE instead of sizeof(PyBytesObject) saves + 3 or 7 bytes per bytes object allocation on a typical system. +*/ +#define _PyBytesObject_SIZE (offsetof(PyBytesObject, ob_sval) + 1) - /* Minimum number of allocated bytes, - incremented by _PyBytesWriter_Prepare() */ - Py_ssize_t min_size; +/* --- PyBytesWriter ------------------------------------------------------ */ - /* If non-zero, use a bytearray instead of a bytes object for buffer. */ +struct PyBytesWriter { + char small_buffer[256]; + PyObject *obj; + Py_ssize_t size; int use_bytearray; - - /* If non-zero, overallocate the buffer (default: 0). - This flag must be zero if use_bytearray is non-zero. */ int overallocate; +}; + +// Export for '_testcapi' shared extension +PyAPI_FUNC(PyBytesWriter*) _PyBytesWriter_CreateByteArray(Py_ssize_t size); - /* Stack buffer */ - int use_small_buffer; - char small_buffer[512]; -} _PyBytesWriter; - -/* Initialize a bytes writer - - By default, the overallocation is disabled. Set the overallocate attribute - to control the allocation of the buffer. - - Export _PyBytesWriter API for '_pickle' shared extension. */ -PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer); - -/* Get the buffer content and reset the writer. - Return a bytes object, or a bytearray object if use_bytearray is non-zero. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer, - void *str); - -/* Deallocate memory of a writer (clear its internal buffer). */ -PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer); - -/* Allocate the buffer to write size bytes. - Return the pointer to the beginning of buffer data. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, - Py_ssize_t size); - -/* Ensure that the buffer is large enough to write *size* bytes. - Add size to the writer minimum size (min_size attribute). - - str is the current pointer inside the buffer. - Return the updated current pointer inside the buffer. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, - void *str, - Py_ssize_t size); - -/* Resize the buffer to make it larger. - The new buffer may be larger than size bytes because of overallocation. - Return the updated current pointer inside the buffer. - Raise an exception and return NULL on error. - - Note: size must be greater than the number of allocated bytes in the writer. - - This function doesn't use the writer minimum size (min_size attribute). - - See also _PyBytesWriter_Prepare(). - */ -PyAPI_FUNC(void*) _PyBytesWriter_Resize(_PyBytesWriter *writer, - void *str, - Py_ssize_t size); - -/* Write bytes. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, - void *str, - const void *bytes, - Py_ssize_t size); +static inline Py_ssize_t +_PyBytesWriter_GetSize(PyBytesWriter *writer) +{ + return writer->size; +} + +static inline char* +_PyBytesWriter_GetData(PyBytesWriter *writer) +{ + if (writer->obj == NULL) { + return writer->small_buffer; + } + else if (writer->use_bytearray) { + return PyByteArray_AS_STRING(writer->obj); + } + else { + return PyBytes_AS_STRING(writer->obj); + } +} #ifdef __cplusplus } diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index 32ac3d17f220778..a9db8860e91c06c 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -64,39 +64,14 @@ PyAPI_FUNC(PyObject*) _PyObject_CallMethod( PyObject *name, const char *format, ...); -extern PyObject* _PyObject_CallMethodIdObjArgs( - PyObject *obj, - _Py_Identifier *name, - ...); - -static inline PyObject * -_PyObject_VectorcallMethodId( - _Py_Identifier *name, PyObject *const *args, - size_t nargsf, PyObject *kwnames) -{ - PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ - if (!oname) { - return _Py_NULL; - } - return PyObject_VectorcallMethod(oname, args, nargsf, kwnames); -} - -static inline PyObject * -_PyObject_CallMethodIdNoArgs(PyObject *self, _Py_Identifier *name) -{ - size_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; - return _PyObject_VectorcallMethodId(name, &self, nargsf, _Py_NULL); -} - -static inline PyObject * -_PyObject_CallMethodIdOneArg(PyObject *self, _Py_Identifier *name, PyObject *arg) -{ - PyObject *args[2] = {self, arg}; - size_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET; - assert(arg != NULL); - return _PyObject_VectorcallMethodId(name, args, nargsf, _Py_NULL); -} +extern PyObject *_PyObject_VectorcallPrepend( + PyThreadState *tstate, + PyObject *callable, + PyObject *arg, + PyObject *const *args, + size_t nargsf, + PyObject *kwnames); /* === Vectorcall protocol (PEP 590) ============================= */ @@ -186,17 +161,18 @@ _PyObject_CallNoArgs(PyObject *func) { } -extern PyObject *const * +PyAPI_FUNC(PyObject *const *) _PyStack_UnpackDict(PyThreadState *tstate, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, PyObject **p_kwnames); -extern void _PyStack_UnpackDict_Free( +// Exported for external JIT support +PyAPI_FUNC(void) _PyStack_UnpackDict_Free( PyObject *const *stack, Py_ssize_t nargs, PyObject *kwnames); -extern void _PyStack_UnpackDict_FreeNoDecRef( +PyAPI_FUNC(void) _PyStack_UnpackDict_FreeNoDecRef( PyObject *const *stack, PyObject *kwnames); diff --git a/Include/internal/pycore_cell.h b/Include/internal/pycore_cell.h index cef01e80514f4b1..d0d45a2343654fa 100644 --- a/Include/internal/pycore_cell.h +++ b/Include/internal/pycore_cell.h @@ -53,7 +53,7 @@ _PyCell_GetStackRef(PyCellObject *cell) { PyObject *value; #ifdef Py_GIL_DISABLED - value = _Py_atomic_load_ptr(&cell->ob_ref); + value = _PyObject_CAST(_Py_atomic_load_ptr(&cell->ob_ref)); if (value == NULL) { return PyStackRef_NULL; } diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index cc2defbdf77821c..06c4ca1619d7ce1 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -22,8 +22,10 @@ struct _ceval_runtime_state; // Export for '_lsprof' shared extension PyAPI_FUNC(int) _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); +extern int _PyEval_SetProfileAllThreads(PyInterpreterState *interp, Py_tracefunc func, PyObject *arg); extern int _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); +extern int _PyEval_SetTraceAllThreads(PyInterpreterState *interp, Py_tracefunc func, PyObject *arg); extern int _PyEval_SetOpcodeTrace(PyFrameObject *f, bool enable); @@ -31,8 +33,6 @@ extern int _PyEval_SetOpcodeTrace(PyFrameObject *f, bool enable); // Export for 'array' shared extension PyAPI_FUNC(PyObject*) _PyEval_GetBuiltin(PyObject *); -extern PyObject* _PyEval_GetBuiltinId(_Py_Identifier *); - extern void _PyEval_SetSwitchInterval(unsigned long microseconds); extern unsigned long _PyEval_GetSwitchInterval(void); @@ -94,7 +94,7 @@ typedef struct { void* (*init_state)(void); // Callback to register every trampoline being created void (*write_state)(void* state, const void *code_addr, - unsigned int code_size, PyCodeObject* code); + size_t code_size, PyCodeObject* code); // Callback to free the trampoline state int (*free_state)(void* state); } _PyPerf_Callbacks; @@ -103,12 +103,15 @@ extern int _PyPerfTrampoline_SetCallbacks(_PyPerf_Callbacks *); extern void _PyPerfTrampoline_GetCallbacks(_PyPerf_Callbacks *); extern int _PyPerfTrampoline_Init(int activate); extern int _PyPerfTrampoline_Fini(void); -extern void _PyPerfTrampoline_FreeArenas(void); extern int _PyIsPerfTrampolineActive(void); extern PyStatus _PyPerfTrampoline_AfterFork_Child(void); #ifdef PY_HAVE_PERF_TRAMPOLINE extern _PyPerf_Callbacks _Py_perfmap_callbacks; extern _PyPerf_Callbacks _Py_perfmap_jit_callbacks; +extern void _PyPerfJit_WriteNamedCode(const void *code_addr, + size_t code_size, + const char *entry, + const char *filename); #endif static inline PyObject* @@ -121,6 +124,15 @@ _PyEval_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwfl return tstate->interp->eval_frame(tstate, frame, throwflag); } +#ifdef _Py_TIER2 +_Py_CODEUNIT *_PyTier2Interpreter( + struct _PyExecutorObject *current_executor, _PyInterpreterFrame *frame, + _PyStackRef *stack_pointer, PyThreadState *tstate +); +#endif + +extern _PyJitEntryFuncPtr _Py_jit_entry; + extern PyObject* _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, @@ -196,10 +208,17 @@ extern void _PyEval_DeactivateOpCache(void); /* --- _Py_EnterRecursiveCall() ----------------------------------------- */ -static inline int _Py_MakeRecCheck(PyThreadState *tstate) { +static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + // Possible overflow if stack pointer is beyond the soft limit. + // _Py_CheckRecursiveCall will check for corner cases and + // report an error if there is an overflow. +#if _Py_STACK_GROWS_DOWN return here_addr < _tstate->c_stack_soft_limit; +#else + return here_addr > _tstate->c_stack_soft_limit; +#endif } // Export for '_json' shared extension, used via _Py_EnterRecursiveCall() @@ -208,12 +227,12 @@ PyAPI_FUNC(int) _Py_CheckRecursiveCall( PyThreadState *tstate, const char *where); -int _Py_CheckRecursiveCallPy( +PyAPI_FUNC(int) _Py_CheckRecursiveCallPy( PyThreadState *tstate); static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate, const char *where) { - return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where)); + return (_Py_ReachedRecursionLimit(tstate) && _Py_CheckRecursiveCall(tstate, where)); } static inline int _Py_EnterRecursiveCall(const char *where) { @@ -227,12 +246,10 @@ static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) { PyAPI_FUNC(void) _Py_InitializeRecursionLimits(PyThreadState *tstate); -static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate) { - uintptr_t here_addr = _Py_get_machine_stack_pointer(); - _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - assert(_tstate->c_stack_hard_limit != 0); - return here_addr <= _tstate->c_stack_soft_limit; -} +// Export for test_peg_generator +PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin( + PyThreadState *tstate, + int margin_count); static inline void _Py_LeaveRecursiveCall(void) { } @@ -255,6 +272,9 @@ PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); and asynchronous exception */ PyAPI_FUNC(int) _Py_HandlePending(PyThreadState *tstate); +/* Raise exception set by PyThreadState_SetAsyncExc, if any */ +PyAPI_FUNC(int) _PyEval_RaiseAsyncExc(PyThreadState *tstate); + extern PyObject * _PyEval_GetFrameLocals(void); typedef PyObject *(*conversion_func)(PyObject *); @@ -277,12 +297,25 @@ PyAPI_FUNC(int) _PyEval_ExceptionGroupMatch(_PyInterpreterFrame *, PyObject* exc PyAPI_FUNC(void) _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg); PyAPI_FUNC(void) _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj); PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg); -PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); +PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs, PyObject *dupkey); PyAPI_FUNC(PyObject *) _PyEval_ImportFrom(PyThreadState *, PyObject *, PyObject *); -PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *); + +PyAPI_FUNC(PyObject *) _PyEval_LazyImportName( + PyThreadState *tstate, PyObject *builtins, PyObject *globals, + PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level, + int lazy); +PyAPI_FUNC(PyObject *) _PyEval_LazyImportFrom( + PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *v, PyObject *name); +PyAPI_FUNC(PyObject *) _PyEval_ImportName( + PyThreadState *tstate, PyObject *builtins, PyObject *globals, + PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level); +PyObject * _PyEval_ImportNameWithImport( + PyThreadState *tstate, PyObject *import_func, PyObject *globals, + PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level); PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); +PyAPI_FUNC(bool) _PyEval_NoToolsForUnwind(PyThreadState *tstate, _PyInterpreterFrame *frame); PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, _PyStackRef *sp); PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch); @@ -295,6 +328,7 @@ PyAPI_FUNC(PyObject *) _PyEval_GetAwaitable(PyObject *iterable, int oparg); PyAPI_FUNC(PyObject *) _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name); PyAPI_FUNC(int) _Py_Check_ArgsIterable(PyThreadState *tstate, PyObject *func, PyObject *args); +PyAPI_FUNC(_PyStackRef) _PyEval_GetIter(_PyStackRef iterable, _PyStackRef *null_or_index, int yield_from); /* * Indicate whether a special method of given 'oparg' can use the (improved) @@ -344,8 +378,6 @@ _Py_eval_breaker_bit_is_set(PyThreadState *tstate, uintptr_t bit) void _Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); -PyAPI_FUNC(_PyStackRef) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value); - #ifndef Py_SUPPORTS_REMOTE_DEBUG #if defined(__APPLE__) #include @@ -373,6 +405,87 @@ _PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame #define SPECIAL___AEXIT__ 3 #define SPECIAL_MAX 3 +PyAPI_DATA(const _Py_CODEUNIT *) _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR; + +/* Helper functions for large uops */ + +PyAPI_FUNC(PyObject *) +_Py_VectorCall_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args, + _PyStackRef kwnames); + +PyAPI_FUNC(PyObject*) +_Py_VectorCallInstrumentation_StackRefSteal( + _PyStackRef callable, + _PyStackRef* arguments, + int total_args, + _PyStackRef kwnames, + bool call_instrumentation, + _PyInterpreterFrame* frame, + _Py_CODEUNIT* this_instr, + PyThreadState* tstate); + +PyAPI_FUNC(PyObject *) +_Py_BuiltinCallFast_StackRef( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_BuiltinCallFastWithKeywords_StackRef( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_PyCallMethodDescriptorFast_StackRef( + _PyStackRef callable, + PyCFunctionFast cfunc, + PyObject *self, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_PyCallMethodDescriptorFastWithKeywords_StackRef( + _PyStackRef callable, + PyCFunctionFastWithKeywords cfunc, + PyObject *self, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_CallBuiltinClass_StackRef( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_BuildString_StackRefSteal( + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_BuildMap_StackRefSteal( + _PyStackRef *arguments, + int half_args); + +PyAPI_FUNC(void) +_Py_assert_within_stack_bounds( + _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + const char *filename, int lineno); + +PyAPI_FUNC(_PyStackRef) +_Py_LoadAttr_StackRefSteal( + PyThreadState *tstate, _PyStackRef owner, + PyObject *name, _PyStackRef *self_or_null); + +// Like PyMapping_GetOptionalItem, but returns the PyObject* instead of taking +// it as an out parameter. This helps MSVC's escape analysis when used with +// tail calling. +PyAPI_FUNC(PyObject*) _PyMapping_GetOptionalItem2(PyObject* obj, PyObject* key, int* err); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 8e1415f27b63f3d..5b1fddbe15b98b8 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -141,6 +141,12 @@ typedef struct { #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache) +typedef struct { + _Py_BackoffCounter counter; +} _PyGetIterCache; + +#define INLINE_CACHE_ENTRIES_GET_ITER CACHE_ENTRIES(_PyGetIterCache) + typedef struct { _Py_BackoffCounter counter; } _PySendCache; @@ -160,6 +166,12 @@ typedef struct { #define INLINE_CACHE_ENTRIES_CONTAINS_OP CACHE_ENTRIES(_PyContainsOpCache) +typedef struct { + _Py_BackoffCounter counter; +} _PyCallFunctionExCache; + +#define INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX CACHE_ENTRIES(_PyCallFunctionExCache) + /* "Locals plus" for a code object is the set of locals + cell vars + * free vars. This relates to variable names as well as offsets into * the "fast locals" storage array of execution frames. The compiler @@ -262,7 +274,7 @@ extern PyObject* _PyCode_GetFreevars(PyCodeObject *); extern PyObject* _PyCode_GetCode(PyCodeObject *); /** API for initializing the line number tables. */ -extern int _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds); +PyAPI_FUNC(int) _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds); /** Out of process API for initializing the location table. */ extern void _PyLineTable_InitAddressRange( @@ -272,52 +284,53 @@ extern void _PyLineTable_InitAddressRange( PyCodeAddressRange *range); /** API for traversing the line number table. */ -extern int _PyLineTable_NextAddressRange(PyCodeAddressRange *range); +PyAPI_FUNC(int) _PyLineTable_NextAddressRange(PyCodeAddressRange *range); extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); +// Similar to PyCode_Addr2Line(), but return -1 if the code object is invalid +// and can be called without an attached tstate. Used by dump_frame() in +// Python/traceback.c. The function uses heuristics to detect freed memory, +// it's not 100% reliable. +extern int _PyCode_SafeAddr2Line(PyCodeObject *co, int addr); + + /** API for executors */ extern void _PyCode_Clear_Executors(PyCodeObject *code); -#ifdef Py_GIL_DISABLED -// gh-115999 tracks progress on addressing this. -#define ENABLE_SPECIALIZATION 0 -// Use this to enable specialization families once they are thread-safe. All -// uses will be replaced with ENABLE_SPECIALIZATION once all families are -// thread-safe. -#define ENABLE_SPECIALIZATION_FT 1 -#else #define ENABLE_SPECIALIZATION 1 -#define ENABLE_SPECIALIZATION_FT ENABLE_SPECIALIZATION -#endif -/* Specialization functions */ +/* Specialization functions, these are exported only for other re-generated + * interpreters to call */ -extern void _Py_Specialize_LoadSuperAttr(_PyStackRef global_super, _PyStackRef cls, +PyAPI_FUNC(void) _Py_Specialize_LoadSuperAttr(_PyStackRef global_super, _PyStackRef cls, _Py_CODEUNIT *instr, int load_method); -extern void _Py_Specialize_LoadAttr(_PyStackRef owner, _Py_CODEUNIT *instr, +PyAPI_FUNC(void) _Py_Specialize_LoadAttr(_PyStackRef owner, _Py_CODEUNIT *instr, PyObject *name); -extern void _Py_Specialize_StoreAttr(_PyStackRef owner, _Py_CODEUNIT *instr, +PyAPI_FUNC(void) _Py_Specialize_StoreAttr(_PyStackRef owner, _Py_CODEUNIT *instr, PyObject *name); -extern void _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, +PyAPI_FUNC(void) _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name); -extern void _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, +PyAPI_FUNC(void) _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, _Py_CODEUNIT *instr); -extern void _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr, - int nargs); -extern void _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, +PyAPI_FUNC(void) _Py_Specialize_Call(_PyStackRef callable, _PyStackRef self_or_null, + _Py_CODEUNIT *instr, int nargs); +PyAPI_FUNC(void) _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, int nargs); -extern void _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, +PyAPI_FUNC(void) _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg, _PyStackRef *locals); -extern void _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, +PyAPI_FUNC(void) _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_UnpackSequence(_PyStackRef seq, _Py_CODEUNIT *instr, +PyAPI_FUNC(void) _Py_Specialize_UnpackSequence(_PyStackRef seq, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr); -extern void _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr); -extern void _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr); -extern void _Py_GatherStats_GetIter(_PyStackRef iterable); +PyAPI_FUNC(void) _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT *instr, int oparg); +PyAPI_FUNC(void) _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr); +PyAPI_FUNC(void) _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr); +PyAPI_FUNC(void) _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr); +PyAPI_FUNC(void) _Py_GatherStats_GetIter(_PyStackRef iterable); +PyAPI_FUNC(void) _Py_Specialize_CallFunctionEx(_PyStackRef func_st, _Py_CODEUNIT *instr); +PyAPI_FUNC(void) _Py_Specialize_Resume(_Py_CODEUNIT *instr, PyThreadState *tstate, _PyInterpreterFrame *frame); +PyAPI_FUNC(void) _Py_Specialize_GetIter(_PyStackRef iterable, _Py_CODEUNIT *instr); // Utility functions for reading/writing 32/64-bit values in the inline caches. // Great care should be taken to ensure that these functions remain correct and @@ -490,6 +503,18 @@ typedef struct { int oparg; binaryopguardfunc guard; binaryopactionfunc action; + /* Static type of the result, or NULL if unknown. Used by the tier 2 + optimizer to propagate type information through _BINARY_OP_EXTEND. */ + PyTypeObject *result_type; + /* Nonzero iff `action` always returns a freshly allocated object (not + aliased to either operand). Used by the tier 2 optimizer to enable + inplace follow-up ops. */ + int result_unique; + /* Expected types of the left and right operands. Used by the tier 2 + optimizer to eliminate _GUARD_BINARY_OP_EXTEND when the operand + types are already known. NULL means unknown/don't eliminate. */ + PyTypeObject *lhs_type; + PyTypeObject *rhs_type; } _PyBinaryOpSpecializationDescr; /* Comparison bit masks. */ @@ -512,9 +537,10 @@ typedef struct { #define COMPARISON_NOT_EQUALS (COMPARISON_UNORDERED | COMPARISON_LESS_THAN | COMPARISON_GREATER_THAN) -extern int _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); +PyAPI_FUNC(int) _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); -extern _Py_CODEUNIT _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(_Py_CODEUNIT) _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); extern int _PyInstruction_GetLength(PyCodeObject *code, int offset); @@ -548,7 +574,7 @@ _PyCode_GetTLBCFast(PyThreadState *tstate, PyCodeObject *co) // Return a pointer to the thread-local bytecode for the current thread, // creating it if necessary. -extern _Py_CODEUNIT *_PyCode_GetTLBC(PyCodeObject *co); +PyAPI_FUNC(_Py_CODEUNIT *) _PyCode_GetTLBC(PyCodeObject *co); // Reserve an index for the current thread into thread-local bytecode // arrays @@ -660,6 +686,15 @@ PyAPI_FUNC(int) _PyCode_VerifyStateless( PyAPI_FUNC(int) _PyCode_CheckPureFunction(PyCodeObject *, const char **); PyAPI_FUNC(int) _PyCode_ReturnsOnlyNone(PyCodeObject *); +/* Create a comparable key used to compare constants taking in account the + * object type. It is used to make sure types are not coerced (e.g., float and + * complex) _and_ to distinguish 0.0 from -0.0 e.g. on IEEE platforms + * + * Return (type(obj), obj, ...): a tuple with variable size (at least 2 items) + * depending on the type and the value. The type is the first item to not + * compare bytes and str which can raise a BytesWarning exception. */ +extern PyObject* _PyCode_ConstantKey(PyObject *obj); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h index 52dca1362592d69..bfa10eadf735731 100644 --- a/Include/internal/pycore_codecs.h +++ b/Include/internal/pycore_codecs.h @@ -45,7 +45,7 @@ extern int _PyCodec_UnregisterError(const char *name); in Python 3.5+? */ -extern PyObject* _PyCodec_LookupTextEncoding( +PyAPI_FUNC(PyObject*) _PyCodec_LookupTextEncoding( const char *encoding, const char *alternate_command); diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index c18e04bf67a5df4..911cc1f10f15131 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -32,7 +32,8 @@ PyAPI_FUNC(PyCodeObject*) _PyAST_Compile( PyObject *filename, PyCompilerFlags *flags, int optimize, - struct _arena *arena); + struct _arena *arena, + PyObject *module); /* AST preprocessing */ extern int _PyCompile_AstPreprocess( @@ -41,7 +42,8 @@ extern int _PyCompile_AstPreprocess( PyCompilerFlags *flags, int optimize, struct _arena *arena, - int syntax_check_only); + int syntax_check_only, + PyObject *module); extern int _PyAST_Preprocess( struct _mod *, @@ -49,7 +51,9 @@ extern int _PyAST_Preprocess( PyObject *filename, int optimize, int ff_features, - int syntax_check_only); + int syntax_check_only, + int enable_warnings, + PyObject *module); typedef struct { @@ -127,6 +131,7 @@ int _PyCompile_PushFBlock(struct _PyCompiler *c, _Py_SourceLocation loc, void _PyCompile_PopFBlock(struct _PyCompiler *c, enum _PyCompile_FBlockType t, _PyJumpTargetLabel block_label); _PyCompile_FBlockInfo *_PyCompile_TopFBlock(struct _PyCompiler *c); +bool _PyCompile_InExceptionHandler(struct _PyCompiler *c); int _PyCompile_EnterScope(struct _PyCompiler *c, identifier name, int scope_type, void *key, int lineno, PyObject *private, diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index c77ef7910c09aa6..a833f790a621b1d 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -55,5 +55,8 @@ struct _pycontexttokenobject { // Export for '_testcapi' shared extension PyAPI_FUNC(PyObject*) _PyContext_NewHamtForTests(void); +PyAPI_FUNC(int) _PyContext_Enter(PyThreadState *ts, PyObject *octx); +PyAPI_FUNC(int) _PyContext_Exit(PyThreadState *ts, PyObject *octx); + #endif /* !Py_INTERNAL_CONTEXT_H */ diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 2601de40737e85c..51d99d74ca159f6 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -32,7 +32,7 @@ extern "C" { const bool _should_lock_cs = PyList_CheckExact(_orig_seq); \ PyCriticalSection _cs; \ if (_should_lock_cs) { \ - _PyCriticalSection_Begin(&_cs, _orig_seq); \ + PyCriticalSection_Begin(&_cs, _orig_seq); \ } # define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() \ @@ -50,6 +50,15 @@ extern "C" { // Asserts that the mutex for the given object is locked. The mutex must // be held by the top-most critical section otherwise there's the // possibility that the mutex would be swalled out in some code paths. +// +// NOTE: We use Py_REFCNT(op) != 1 instead of +// !PyUnstable_Object_IsUniquelyReferenced(op) because the free threading +// GC can change an object's ob_tid (it overwrites ob_tid and later +// restores it from the mimalloc segment). This means +// PyUnstable_Object_IsUniquelyReferenced() may spuriously return false +// after a GC collection, even though the thread may still have exclusive +// access to the object. The refcount check is a looser but still catches +// most misuses. #ifdef Py_DEBUG # define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) \ @@ -77,10 +86,10 @@ _PyCriticalSection_Resume(PyThreadState *tstate); // (private) slow path for locking the mutex PyAPI_FUNC(void) -_PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m); +_PyCriticalSection_BeginSlow(PyThreadState *tstate, PyCriticalSection *c, PyMutex *m); PyAPI_FUNC(void) -_PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, +_PyCriticalSection2_BeginSlow(PyThreadState *tstate, PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, int is_m1_locked); PyAPI_FUNC(void) @@ -95,34 +104,30 @@ _PyCriticalSection_IsActive(uintptr_t tag) } static inline void -_PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m) +_PyCriticalSection_BeginMutex(PyThreadState *tstate, PyCriticalSection *c, PyMutex *m) { if (PyMutex_LockFast(m)) { - PyThreadState *tstate = _PyThreadState_GET(); c->_cs_mutex = m; c->_cs_prev = tstate->critical_section; tstate->critical_section = (uintptr_t)c; } else { - _PyCriticalSection_BeginSlow(c, m); + _PyCriticalSection_BeginSlow(tstate, c, m); } } -#define PyCriticalSection_BeginMutex _PyCriticalSection_BeginMutex static inline void -_PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) +_PyCriticalSection_Begin(PyThreadState *tstate, PyCriticalSection *c, PyObject *op) { - _PyCriticalSection_BeginMutex(c, &op->ob_mutex); + _PyCriticalSection_BeginMutex(tstate, c, &op->ob_mutex); } -#define PyCriticalSection_Begin _PyCriticalSection_Begin // Removes the top-most critical section from the thread's stack of critical // sections. If the new top-most critical section is inactive, then it is // resumed. static inline void -_PyCriticalSection_Pop(PyCriticalSection *c) +_PyCriticalSection_Pop(PyThreadState *tstate, PyCriticalSection *c) { - PyThreadState *tstate = _PyThreadState_GET(); uintptr_t prev = c->_cs_prev; tstate->critical_section = prev; @@ -132,7 +137,7 @@ _PyCriticalSection_Pop(PyCriticalSection *c) } static inline void -_PyCriticalSection_End(PyCriticalSection *c) +_PyCriticalSection_End(PyThreadState *tstate, PyCriticalSection *c) { // If the mutex is NULL, we used the fast path in // _PyCriticalSection_BeginSlow for locks already held in the top-most @@ -141,18 +146,17 @@ _PyCriticalSection_End(PyCriticalSection *c) return; } PyMutex_Unlock(c->_cs_mutex); - _PyCriticalSection_Pop(c); + _PyCriticalSection_Pop(tstate, c); } -#define PyCriticalSection_End _PyCriticalSection_End static inline void -_PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) +_PyCriticalSection2_BeginMutex(PyThreadState *tstate, PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) { if (m1 == m2) { // If the two mutex arguments are the same, treat this as a critical // section with a single mutex. c->_cs_mutex2 = NULL; - _PyCriticalSection_BeginMutex(&c->_cs_base, m1); + _PyCriticalSection_BeginMutex(tstate, &c->_cs_base, m1); return; } @@ -167,7 +171,6 @@ _PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) if (PyMutex_LockFast(m1)) { if (PyMutex_LockFast(m2)) { - PyThreadState *tstate = _PyThreadState_GET(); c->_cs_base._cs_mutex = m1; c->_cs_mutex2 = m2; c->_cs_base._cs_prev = tstate->critical_section; @@ -176,29 +179,26 @@ _PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) tstate->critical_section = p; } else { - _PyCriticalSection2_BeginSlow(c, m1, m2, 1); + _PyCriticalSection2_BeginSlow(tstate, c, m1, m2, 1); } } else { - _PyCriticalSection2_BeginSlow(c, m1, m2, 0); + _PyCriticalSection2_BeginSlow(tstate, c, m1, m2, 0); } } -#define PyCriticalSection2_BeginMutex _PyCriticalSection2_BeginMutex static inline void -_PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) +_PyCriticalSection2_Begin(PyThreadState *tstate, PyCriticalSection2 *c, PyObject *a, PyObject *b) { - _PyCriticalSection2_BeginMutex(c, &a->ob_mutex, &b->ob_mutex); + _PyCriticalSection2_BeginMutex(tstate, c, &a->ob_mutex, &b->ob_mutex); } -#define PyCriticalSection2_Begin _PyCriticalSection2_Begin static inline void -_PyCriticalSection2_End(PyCriticalSection2 *c) +_PyCriticalSection2_End(PyThreadState *tstate, PyCriticalSection2 *c) { - // if mutex1 is NULL, we used the fast path in - // _PyCriticalSection_BeginSlow for mutexes that are already held, - // which should only happen when mutex1 and mutex2 were the same mutex, - // and mutex2 should also be NULL. + // if mutex1 is NULL, we used the fast path in either + // _PyCriticalSection_BeginSlow or _PyCriticalSection2_BeginSlow for mutexes + // that are already held, and mutex2 should also be NULL. if (c->_cs_base._cs_mutex == NULL) { assert(c->_cs_mutex2 == NULL); return; @@ -207,9 +207,8 @@ _PyCriticalSection2_End(PyCriticalSection2 *c) PyMutex_Unlock(c->_cs_mutex2); } PyMutex_Unlock(c->_cs_base._cs_mutex); - _PyCriticalSection_Pop(&c->_cs_base); + _PyCriticalSection_Pop(tstate, &c->_cs_base); } -#define PyCriticalSection2_End _PyCriticalSection2_End static inline void _PyCriticalSection_AssertHeld(PyMutex *mutex) @@ -251,6 +250,45 @@ _PyCriticalSection_AssertHeldObj(PyObject *op) #endif } + +#undef Py_BEGIN_CRITICAL_SECTION +# define Py_BEGIN_CRITICAL_SECTION(op) \ + { \ + PyCriticalSection _py_cs; \ + PyThreadState *_cs_tstate = _PyThreadState_GET(); \ + _PyCriticalSection_Begin(_cs_tstate, &_py_cs, _PyObject_CAST(op)) + +#undef Py_BEGIN_CRITICAL_SECTION_MUTEX +# define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ + { \ + PyCriticalSection _py_cs; \ + PyThreadState *_cs_tstate = _PyThreadState_GET(); \ + _PyCriticalSection_BeginMutex(_cs_tstate, &_py_cs, mutex) + +#undef Py_END_CRITICAL_SECTION +# define Py_END_CRITICAL_SECTION() \ + _PyCriticalSection_End(_cs_tstate, &_py_cs); \ + } + +#undef Py_BEGIN_CRITICAL_SECTION2 +# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyThreadState *_cs_tstate = _PyThreadState_GET(); \ + _PyCriticalSection2_Begin(_cs_tstate, &_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) + +#undef Py_BEGIN_CRITICAL_SECTION2_MUTEX +# define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyThreadState *_cs_tstate = _PyThreadState_GET(); \ + _PyCriticalSection2_BeginMutex(_cs_tstate, &_py_cs2, m1, m2) + +#undef Py_END_CRITICAL_SECTION2 +# define Py_END_CRITICAL_SECTION2() \ + _PyCriticalSection2_End(_cs_tstate, &_py_cs2); \ + } + #endif /* Py_GIL_DISABLED */ #ifdef __cplusplus diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 81faffac1941719..bed966681fa1f0c 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -265,6 +265,12 @@ typedef struct { // heap types PyObject *PyExc_NotShareableError; } exceptions; + + // Cached references to pickle.dumps/loads (per-interpreter). + struct { + PyObject *dumps; + PyObject *loads; + } pickle; } _PyXI_state_t; #define _PyXI_GET_GLOBAL_STATE(interp) (&(interp)->runtime->xi) diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 1b59fa2ef600144..18490f98a918a73 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -102,12 +102,23 @@ typedef struct _Py_DebugOffsets { uint64_t next; uint64_t interp; uint64_t current_frame; + uint64_t base_frame; + uint64_t last_profiled_frame; uint64_t thread_id; uint64_t native_thread_id; uint64_t datastack_chunk; uint64_t status; + uint64_t holds_gil; + uint64_t gil_requested; + uint64_t current_exception; + uint64_t exc_state; } thread_state; + // Exception stack item offset + struct { + uint64_t exc_value; + } err_stackitem; + // InterpreterFrame offset; struct _interpreter_frame { uint64_t size; @@ -147,8 +158,16 @@ typedef struct _Py_DebugOffsets { uint64_t tp_name; uint64_t tp_repr; uint64_t tp_flags; + uint64_t tp_basicsize; + uint64_t tp_dictoffset; } type_object; + // PyHeapTypeObject offset; + struct _heap_type_object { + uint64_t size; + uint64_t ht_cached_keys; + } heap_type_object; + // PyTuple object offset; struct _tuple_object { uint64_t size; @@ -204,12 +223,16 @@ typedef struct _Py_DebugOffsets { uint64_t state; uint64_t length; uint64_t asciiobject_size; + uint64_t compactunicodeobject_size; } unicode_object; // GC runtime state offset; struct _gc { uint64_t size; uint64_t collecting; + uint64_t frame; + uint64_t generation_stats_size; + uint64_t generation_stats; } gc; // Generator object offset; @@ -269,10 +292,19 @@ typedef struct _Py_DebugOffsets { .next = offsetof(PyThreadState, next), \ .interp = offsetof(PyThreadState, interp), \ .current_frame = offsetof(PyThreadState, current_frame), \ + .base_frame = offsetof(PyThreadState, base_frame), \ + .last_profiled_frame = offsetof(PyThreadState, last_profiled_frame), \ .thread_id = offsetof(PyThreadState, thread_id), \ .native_thread_id = offsetof(PyThreadState, native_thread_id), \ .datastack_chunk = offsetof(PyThreadState, datastack_chunk), \ .status = offsetof(PyThreadState, _status), \ + .holds_gil = offsetof(PyThreadState, holds_gil), \ + .gil_requested = offsetof(PyThreadState, gil_requested), \ + .current_exception = offsetof(PyThreadState, current_exception), \ + .exc_state = offsetof(PyThreadState, exc_state), \ + }, \ + .err_stackitem = { \ + .exc_value = offsetof(_PyErr_StackItem, exc_value), \ }, \ .interpreter_frame = { \ .size = sizeof(_PyInterpreterFrame), \ @@ -306,6 +338,12 @@ typedef struct _Py_DebugOffsets { .tp_name = offsetof(PyTypeObject, tp_name), \ .tp_repr = offsetof(PyTypeObject, tp_repr), \ .tp_flags = offsetof(PyTypeObject, tp_flags), \ + .tp_basicsize = offsetof(PyTypeObject, tp_basicsize), \ + .tp_dictoffset = offsetof(PyTypeObject, tp_dictoffset), \ + }, \ + .heap_type_object = { \ + .size = sizeof(PyHeapTypeObject), \ + .ht_cached_keys = offsetof(PyHeapTypeObject, ht_cached_keys), \ }, \ .tuple_object = { \ .size = sizeof(PyTupleObject), \ @@ -347,10 +385,14 @@ typedef struct _Py_DebugOffsets { .state = offsetof(PyUnicodeObject, _base._base.state), \ .length = offsetof(PyUnicodeObject, _base._base.length), \ .asciiobject_size = sizeof(PyASCIIObject), \ + .compactunicodeobject_size = sizeof(PyCompactUnicodeObject), \ }, \ .gc = { \ .size = sizeof(struct _gc_runtime_state), \ .collecting = offsetof(struct _gc_runtime_state, collecting), \ + .frame = offsetof(struct _gc_runtime_state, frame), \ + .generation_stats_size = sizeof(struct gc_stats), \ + .generation_stats = offsetof(struct _gc_runtime_state, generation_stats), \ }, \ .gen_object = { \ .size = sizeof(PyGenObject), \ @@ -368,7 +410,7 @@ typedef struct _Py_DebugOffsets { .remote_debugging_enabled = offsetof(PyInterpreterState, config.remote_debug), \ .debugger_pending_call = offsetof(_PyRemoteDebuggerSupport, debugger_pending_call), \ .debugger_script_path = offsetof(_PyRemoteDebuggerSupport, debugger_script_path), \ - .debugger_script_path_size = Py_MAX_SCRIPT_PATH_SIZE, \ + .debugger_script_path_size = _Py_MAX_SCRIPT_PATH_SIZE, \ }, \ } diff --git a/Include/internal/pycore_descrobject.h b/Include/internal/pycore_descrobject.h index 3cec59a68a3d2b2..6143f82176a1f2f 100644 --- a/Include/internal/pycore_descrobject.h +++ b/Include/internal/pycore_descrobject.h @@ -22,6 +22,8 @@ typedef propertyobject _PyPropertyObject; extern PyTypeObject _PyMethodWrapper_Type; +extern void *_PyMember_GetOffset(PyObject *, PyMemberDef *); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 25bb224921aa8bc..9e11c0bb546b685 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -30,23 +30,29 @@ PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyDict_DelItem_KnownHash(PyObject *mp, PyObject *key, Py_hash_t hash); + +// Exported for external JIT support +PyAPI_FUNC(int) _PyDict_DelItem_KnownHash_LockHeld(PyObject *mp, PyObject *key, + Py_hash_t hash); + extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); -// "Id" variants -extern PyObject* _PyDict_GetItemIdWithError(PyObject *dp, - _Py_Identifier *key); -extern int _PyDict_ContainsId(PyObject *, _Py_Identifier *); -extern int _PyDict_SetItemId(PyObject *dp, _Py_Identifier *key, PyObject *item); -extern int _PyDict_DelItemId(PyObject *mp, _Py_Identifier *key); +extern void _PyDict_ClearKeysVersionLockHeld(PyObject *mp); extern int _PyDict_Next( PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash); extern int _PyDict_HasOnlyStringKeys(PyObject *mp); +PyAPI_FUNC(PyObject *) _PyDict_Subscript(PyObject *self, PyObject *key); +PyAPI_FUNC(PyObject *) _PyDict_SubscriptKnownHash(PyObject *self, PyObject *key, Py_hash_t hash); +PyAPI_FUNC(int) _PyDict_StoreSubscript(PyObject *self, PyObject *key, PyObject *value); + // Export for '_ctypes' shared extension PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); +extern Py_ssize_t _PyDict_SizeOf_LockHeld(PyDictObject *); + #define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) /* Like PyDict_Merge, but override can be 0, 1 or 2. If override is 0, @@ -54,7 +60,7 @@ PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); of a key wins, if override is 2, a KeyError with conflicting key as argument is raised. */ -PyAPI_FUNC(int) _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); +PyAPI_FUNC(int) _PyDict_MergeUniq(PyObject *mp, PyObject *other, PyObject **dupkey); extern void _PyDict_DebugMallocStats(FILE *out); @@ -84,11 +90,19 @@ typedef struct { } PyDictUnicodeEntry; extern PyDictKeysObject *_PyDict_NewKeysForClass(PyHeapTypeObject *); +extern void _PyDict_RemoveKeysForClass(PyHeapTypeObject *); +extern void _PyDict_SplitKeysInvalidated(PyDictKeysObject* keys); extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); +/* Implementations of the `|` and `|=` operators for dict, used by the + * specializing interpreter. */ +extern PyObject *_PyDict_Or(PyObject *self, PyObject *other); +extern PyObject *_PyDict_IOr(PyObject *self, PyObject *other); + /* Gets a version number unique to the current state of the keys of dict, if possible. - * Returns the version number, or zero if it was not possible to get a version number. */ -extern uint32_t _PyDictKeys_GetVersionForCurrentState( + * Returns the version number, or zero if it was not possible to get a version number. + * Exported for external JIT support */ +PyAPI_FUNC(uint32_t) _PyDictKeys_GetVersionForCurrentState( PyInterpreterState *interp, PyDictKeysObject *dictkeys); /* Gets a version number unique to the current state of the keys of dict, if possible. @@ -98,8 +112,9 @@ extern uint32_t _PyDictKeys_GetVersionForCurrentState( * * The caller must hold the per-object lock on dict. * - * Returns the version number, or zero if it was not possible to get a version number. */ -extern uint32_t _PyDict_GetKeysVersionForCurrentState( + * Returns the version number, or zero if it was not possible to get a version number. + * Exported for external JIT support */ +PyAPI_FUNC(uint32_t) _PyDict_GetKeysVersionForCurrentState( PyInterpreterState *interp, PyDictObject *dict); extern size_t _PyDict_KeysSize(PyDictKeysObject *keys); @@ -108,13 +123,18 @@ extern void _PyDictKeys_DecRef(PyDictKeysObject *keys); /* _Py_dict_lookup() returns index of entry which can be used like DK_ENTRIES(dk)[index]. * -1 when no entry found, -3 when compare raises error. + * Exported for external JIT support */ -extern Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); +PyAPI_FUNC(Py_ssize_t) _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); extern Py_ssize_t _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); extern Py_ssize_t _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject *key, Py_hash_t hash, _PyStackRef *value_addr); -extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *); -extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); +extern int _PyDict_GetMethodStackRef(PyDictObject *dict, PyObject *name, _PyStackRef *method); + +// Exported for external JIT support +PyAPI_FUNC(Py_ssize_t) _PyDict_LookupIndexAndValue(PyDictObject *, PyObject *, PyObject **); +PyAPI_FUNC(Py_ssize_t) _PyDict_LookupIndex(PyDictObject *, PyObject *); +PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); /* Look up a string key in an all unicode dict keys, assign the keys object a version, and * store it in version. @@ -123,9 +143,11 @@ extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject * strings. * * Returns DKIX_EMPTY if the key is not present. + * + * Exported for external JIT support */ -extern Py_ssize_t _PyDictKeys_StringLookupAndVersion(PyDictKeysObject* dictkeys, PyObject *key, uint32_t *version); -extern Py_ssize_t _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key); +PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookupAndVersion(PyDictKeysObject* dictkeys, PyObject *key, uint32_t *version); +PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key); PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); PyAPI_FUNC(void) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *); @@ -134,15 +156,17 @@ extern PyObject *_PyDict_LoadBuiltinsFromGlobals(PyObject *globals); /* Consumes references to key and value */ PyAPI_FUNC(int) _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); -extern int _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value); +PyAPI_FUNC(int) _PyDict_SetItem_Take2_KnownHash(PyDictObject *op, PyObject *key, PyObject *value, Py_hash_t hash); +// Exported for external JIT support +PyAPI_FUNC(int) _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyDict_SetItem_KnownHash_LockHeld(PyDictObject *mp, PyObject *key, PyObject *value, Py_hash_t hash); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyDict_GetItemRef_KnownHash_LockHeld(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); -extern int _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); +PyAPI_FUNC(int) _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); extern int _PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject **result); -extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *name, PyObject *value); +PyAPI_FUNC(int) _PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *name, PyObject *value); extern int _PyDict_Pop_KnownHash( PyDictObject *dict, @@ -156,6 +180,9 @@ extern void _PyDict_Clear_LockHeld(PyObject *op); PyAPI_FUNC(void) _PyDict_EnsureSharedOnRead(PyDictObject *mp); #endif +// Export for '_elementtree' shared extension +PyAPI_FUNC(PyObject*) _PyDict_CopyAsDict(PyObject *op); + #define DKIX_EMPTY (-1) #define DKIX_DUMMY (-2) /* Used internally */ #define DKIX_ERROR (-3) @@ -214,6 +241,17 @@ struct _dictkeysobject { see the DK_ENTRIES() / DK_UNICODE_ENTRIES() functions below */ }; +struct _instancekeysobject { + PyTypeObject* dsk_owning_type; + struct _dictkeysobject dsk_keys; +}; + +static inline struct _instancekeysobject *_PyDictKeys_AsSharedKeys(struct _dictkeysobject *keys) +{ + assert(keys->dk_kind == DICT_KEYS_SPLIT); + return _Py_CONTAINER_OF(keys, struct _instancekeysobject, dsk_keys); +} + /* This must be no more than 250, for the prefix size to fit in one byte. */ #define SHARED_KEYS_MAX_SIZE 30 #define NEXT_LOG2_SHARED_KEYS_MAX_SIZE 6 @@ -262,6 +300,13 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DICT_UNIQUE_ID_SHIFT (32) #define DICT_UNIQUE_ID_MAX ((UINT64_C(1) << (64 - DICT_UNIQUE_ID_SHIFT)) - 1) +/* The first three dict watcher IDs are reserved for CPython, + * so we don't need to check that they haven't been used */ +#define BUILTINS_WATCHER_ID 0 +#define GLOBALS_WATCHER_ID 1 +#define MODULE_WATCHER_ID 2 +#define FIRST_AVAILABLE_WATCHER 3 + PyAPI_FUNC(void) _PyDict_SendEvent(int watcher_bits, @@ -271,14 +316,13 @@ _PyDict_SendEvent(int watcher_bits, PyObject *value); static inline void -_PyDict_NotifyEvent(PyInterpreterState *interp, - PyDict_WatchEvent event, +_PyDict_NotifyEvent(PyDict_WatchEvent event, PyDictObject *mp, PyObject *key, PyObject *value) { assert(Py_REFCNT((PyObject*)mp) > 0); - int watcher_bits = mp->_ma_watcher_tag & DICT_WATCHER_MASK; + int watcher_bits = FT_ATOMIC_LOAD_UINT64_ACQUIRE(mp->_ma_watcher_tag) & DICT_WATCHER_MASK; if (watcher_bits) { RARE_EVENT_STAT_INC(watched_dict_modification); _PyDict_SendEvent(watcher_bits, event, mp, key, value); @@ -310,6 +354,10 @@ _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix) values->size = size+1; } +// Exported for external JIT support +PyAPI_FUNC(void) +_PyDict_InsertSplitValue(PyDictObject *mp, PyObject *key, PyObject *value, Py_ssize_t ix); + static inline size_t shared_keys_usable_size(PyDictKeysObject *keys) { @@ -354,13 +402,13 @@ PyDictObject *_PyObject_MaterializeManagedDict_LockHeld(PyObject *); static inline Py_ssize_t _PyDict_UniqueId(PyDictObject *mp) { - return (Py_ssize_t)(mp->_ma_watcher_tag >> DICT_UNIQUE_ID_SHIFT); + return (Py_ssize_t)(FT_ATOMIC_LOAD_UINT64_RELAXED(mp->_ma_watcher_tag) >> DICT_UNIQUE_ID_SHIFT); } static inline void _Py_INCREF_DICT(PyObject *op) { - assert(PyDict_Check(op)); + assert(PyAnyDict_Check(op)); Py_ssize_t id = _PyDict_UniqueId((PyDictObject *)op); _Py_THREAD_INCREF_OBJECT(op, id); } @@ -368,7 +416,7 @@ _Py_INCREF_DICT(PyObject *op) static inline void _Py_DECREF_DICT(PyObject *op) { - assert(PyDict_Check(op)); + assert(PyAnyDict_Check(op)); Py_ssize_t id = _PyDict_UniqueId((PyDictObject *)op); _Py_THREAD_DECREF_OBJECT(op, id); } @@ -398,6 +446,15 @@ _Py_DECREF_BUILTINS(PyObject *op) } #endif +/* frozendict */ +typedef struct { + PyDictObject ob_base; + Py_hash_t ma_hash; +} PyFrozenDictObject; + +#define _PyFrozenDictObject_CAST(op) \ + (assert(PyFrozenDict_Check(op)), _Py_CAST(PyFrozenDictObject*, (op))) + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index 11932b8d1e1ab60..bb6fe2625975596 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -13,6 +13,8 @@ extern "C" { struct _Py_dict_state { uint32_t next_keys_version; + PyMutex watcher_mutex; // Protects the watchers array (free-threaded builds) + _PyOnceFlag watcher_setup_once; // One-time optimizer watcher setup PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_emscripten_trampoline.h b/Include/internal/pycore_emscripten_trampoline.h index 16916f1a8eb16cc..e37c53a64f4a723 100644 --- a/Include/internal/pycore_emscripten_trampoline.h +++ b/Include/internal/pycore_emscripten_trampoline.h @@ -27,9 +27,6 @@ #if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) -void -_Py_EmscriptenTrampoline_Init(_PyRuntimeState *runtime); - PyObject* _PyEM_TrampolineCall(PyCFunctionWithKeywords func, PyObject* self, diff --git a/Include/internal/pycore_faulthandler.h b/Include/internal/pycore_faulthandler.h index 78cd657e6ae5aee..9ddd70d39ed0d51 100644 --- a/Include/internal/pycore_faulthandler.h +++ b/Include/internal/pycore_faulthandler.h @@ -42,6 +42,7 @@ struct faulthandler_user_signal { int chain; _Py_sighandler_t previous; PyInterpreterState *interp; + Py_ssize_t max_threads; }; #endif /* FAULTHANDLER_USER */ @@ -57,6 +58,7 @@ struct _faulthandler_runtime_state { void *exc_handler; #endif int c_stack; + Py_ssize_t max_threads; } fatal_error; struct { @@ -68,6 +70,7 @@ struct _faulthandler_runtime_state { int exit; char *header; size_t header_len; + Py_ssize_t max_threads; /* The main thread always holds this lock. It is only released when faulthandler_thread() is interrupted before this thread exits, or at Python exit. */ diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index 317f984188bad83..62501cdaf44f072 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -12,7 +12,6 @@ extern "C" { /* runtime lifecycle */ -extern void _PyFloat_InitState(PyInterpreterState *); extern PyStatus _PyFloat_InitTypes(PyInterpreterState *); extern void _PyFloat_FiniType(PyInterpreterState *); @@ -42,6 +41,15 @@ extern double _Py_parse_inf_or_nan(const char *p, char **endptr); extern int _Py_convert_int_to_double(PyObject **v, double *dbl); +/* Should match endianness of the platform in most (all?) cases. */ + +#ifdef DOUBLE_IS_BIG_ENDIAN_IEEE754 +# define _PY_FLOAT_BIG_ENDIAN 1 +# define _PY_FLOAT_LITTLE_ENDIAN 0 +#else +# define _PY_FLOAT_BIG_ENDIAN 0 +# define _PY_FLOAT_LITTLE_ENDIAN 1 +#endif #ifdef __cplusplus } diff --git a/Include/internal/pycore_flowgraph.h b/Include/internal/pycore_flowgraph.h index 5043260d2fd99f2..fc3ae6b369c54c1 100644 --- a/Include/internal/pycore_flowgraph.h +++ b/Include/internal/pycore_flowgraph.h @@ -27,7 +27,7 @@ int _PyCfg_OptimizeCodeUnit(struct _PyCfgBuilder *g, PyObject *consts, PyObject struct _PyCfgBuilder* _PyCfg_FromInstructionSequence(_PyInstructionSequence *seq); int _PyCfg_ToInstructionSequence(struct _PyCfgBuilder *g, _PyInstructionSequence *seq); int _PyCfg_OptimizedCfgToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_CodeUnitMetadata *umd, - int code_flags, int *stackdepth, int *nlocalsplus, + int *stackdepth, int *nlocalsplus, _PyInstructionSequence *seq); PyCodeObject * diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 8c410e9e2083407..3c9ab99c34ebc64 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -38,23 +38,23 @@ struct _frame { PyObject *_f_frame_data[1]; }; -extern PyFrameObject* _PyFrame_New_NoTrack(PyCodeObject *code); +// Exported for external JIT support +PyAPI_FUNC(PyFrameObject *) _PyFrame_New_NoTrack(PyCodeObject *code); /* other API */ typedef enum _framestate { - FRAME_CREATED = -3, - FRAME_SUSPENDED = -2, - FRAME_SUSPENDED_YIELD_FROM = -1, - FRAME_EXECUTING = 0, - FRAME_COMPLETED = 1, - FRAME_CLEARED = 4 + FRAME_CREATED = 0, + FRAME_SUSPENDED = 1, + FRAME_SUSPENDED_YIELD_FROM = 2, + FRAME_SUSPENDED_YIELD_FROM_LOCKED = 3, + FRAME_EXECUTING = 4, + FRAME_CLEARED = 5 } PyFrameState; -#define FRAME_STATE_SUSPENDED(S) ((S) == FRAME_SUSPENDED || (S) == FRAME_SUSPENDED_YIELD_FROM) -#define FRAME_STATE_FINISHED(S) ((S) >= FRAME_COMPLETED) - +#define FRAME_STATE_SUSPENDED(S) ((S) >= FRAME_SUSPENDED && (S) <= FRAME_SUSPENDED_YIELD_FROM_LOCKED) +#define FRAME_STATE_FINISHED(S) ((S) == FRAME_CLEARED) #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h index f3c9a669ad3512a..3a41ec4b54bb069 100644 --- a/Include/internal/pycore_freelist.h +++ b/Include/internal/pycore_freelist.h @@ -17,15 +17,16 @@ extern "C" { static inline struct _Py_freelists * _Py_freelists_GET(void) { - PyThreadState *tstate = _PyThreadState_GET(); #ifdef Py_DEBUG - _Py_EnsureTstateNotNULL(tstate); + _Py_AssertHoldsTstate(); #endif #ifdef Py_GIL_DISABLED + PyThreadState *tstate = _PyThreadState_GET(); return &((_PyThreadStateImpl*)tstate)->freelists; #else - return &tstate->interp->object_state.freelists; + PyInterpreterState *interp = _PyInterpreterState_GET(); + return &interp->object_state.freelists; #endif } diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h index 59beb92f3f7b9c6..46e2a82ea034560 100644 --- a/Include/internal/pycore_freelist_state.h +++ b/Include/internal/pycore_freelist_state.h @@ -27,6 +27,7 @@ extern "C" { # define Py_futureiters_MAXFREELIST 255 # define Py_object_stack_chunks_MAXFREELIST 4 # define Py_unicode_writers_MAXFREELIST 1 +# define Py_bytes_writers_MAXFREELIST 1 # define Py_pycfunctionobject_MAXFREELIST 16 # define Py_pycmethodobject_MAXFREELIST 16 # define Py_pymethodobjects_MAXFREELIST 20 @@ -61,6 +62,7 @@ struct _Py_freelists { struct _Py_freelist futureiters; struct _Py_freelist object_stack_chunks; struct _Py_freelist unicode_writers; + struct _Py_freelist bytes_writers; struct _Py_freelist pycfunctionobject; struct _Py_freelist pycmethodobject; struct _Py_freelist pymethodobjects; diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 6e1209659565a36..2184f40956d4e24 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -extern PyObject* _PyFunction_Vectorcall( +PyAPI_FUNC(PyObject *) _PyFunction_Vectorcall( PyObject *func, PyObject *const *stack, size_t nargsf, @@ -27,10 +27,10 @@ _PyFunction_IsVersionValid(uint32_t version) return version >= FUNC_VERSION_FIRST_VALID; } -extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); +// Exported for external JIT support +PyAPI_FUNC(uint32_t) _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); PyAPI_FUNC(void) _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version); void _PyFunction_ClearCodeByVersion(uint32_t version); -PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version, PyObject **p_code); extern PyObject *_Py_set_function_type_params( PyThreadState* unused, PyObject *func, PyObject *type_params); @@ -47,6 +47,17 @@ static inline PyObject* _PyFunction_GET_BUILTINS(PyObject *func) { #define _PyFunction_GET_BUILTINS(func) _PyFunction_GET_BUILTINS(_PyObject_CAST(func)) +/* Get the callable wrapped by a classmethod. + Returns a borrowed reference. + The caller must ensure 'cm' is a classmethod object. */ +extern PyObject *_PyClassMethod_GetFunc(PyObject *cm); + +/* Get the callable wrapped by a staticmethod. + Returns a borrowed reference. + The caller must ensure 'sm' is a staticmethod object. */ +extern PyObject *_PyStaticMethod_GetFunc(PyObject *sm); + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index a6519aa086309de..84cbb56a9192156 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -118,21 +118,6 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) { /* Bit 1 is set when the object is in generation which is GCed currently. */ #define _PyGC_PREV_MASK_COLLECTING ((uintptr_t)2) -/* Bit 0 in _gc_next is the old space bit. - * It is set as follows: - * Young: gcstate->visited_space - * old[0]: 0 - * old[1]: 1 - * permanent: 0 - * - * During a collection all objects handled should have the bit set to - * gcstate->visited_space, as objects are moved from the young gen - * and the increment into old[gcstate->visited_space]. - * When object are moved from the pending space, old[gcstate->visited_space^1] - * into the increment, the old space bit is flipped. -*/ -#define _PyGC_NEXT_MASK_OLD_SPACE_1 1 - #define _PyGC_PREV_SHIFT 2 #define _PyGC_PREV_MASK (((uintptr_t) -1) << _PyGC_PREV_SHIFT) @@ -159,13 +144,11 @@ typedef enum { // Lowest bit of _gc_next is used for flags only in GC. // But it is always 0 for normal code. static inline PyGC_Head* _PyGCHead_NEXT(PyGC_Head *gc) { - uintptr_t next = gc->_gc_next & _PyGC_PREV_MASK; + uintptr_t next = gc->_gc_next; return (PyGC_Head*)next; } static inline void _PyGCHead_SET_NEXT(PyGC_Head *gc, PyGC_Head *next) { - uintptr_t unext = (uintptr_t)next; - assert((unext & ~_PyGC_PREV_MASK) == 0); - gc->_gc_next = (gc->_gc_next & ~_PyGC_PREV_MASK) | unext; + gc->_gc_next = (uintptr_t)next; } // Lowest two bits of _gc_prev is used for _PyGC_PREV_MASK_* flags. @@ -205,6 +188,8 @@ static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { #endif } +extern void _Py_ScheduleGC(PyThreadState *tstate); + /* Tell the GC to track this object. * @@ -214,7 +199,7 @@ static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { * ob_traverse method. * * Internal note: interp->gc.generation0->_gc_prev doesn't have any bit flags - * because it's not object header. So we don't use _PyGCHead_PREV() and + * because it's not an object header. So we don't use _PyGCHead_PREV() and * _PyGCHead_SET_PREV() for it to avoid unnecessary bitwise operations. * * See also the public PyObject_GC_Track() function. @@ -238,14 +223,14 @@ static inline void _PyObject_GC_TRACK( "object is in generation which is garbage collected", filename, lineno, __func__); - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyGC_Head *generation0 = &interp->gc.young.head; + struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; + PyGC_Head *generation0 = gcstate->generation0; PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); - uintptr_t not_visited = 1 ^ interp->gc.visited_space; - gc->_gc_next = ((uintptr_t)generation0) | not_visited; + _PyGCHead_SET_NEXT(gc, generation0); generation0->_gc_prev = (uintptr_t)gc; + gcstate->heap_size++; #endif } @@ -280,6 +265,8 @@ static inline void _PyObject_GC_UNTRACK( _PyGCHead_SET_PREV(next, prev); gc->_gc_next = 0; gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED; + struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; + gcstate->heap_size--; #endif } @@ -343,14 +330,14 @@ extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); // Functions to clear types free lists extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); -extern void _Py_ScheduleGC(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); union _PyStackRef; // GC visit callback for tracked interpreter frames -extern int _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg); -extern int _PyGC_VisitStackRef(union _PyStackRef *ref, visitproc visit, void *arg); +// GH-150766: exported for greenlet +PyAPI_FUNC(int) _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg); +PyAPI_FUNC(int) _PyGC_VisitStackRef(union _PyStackRef *ref, visitproc visit, void *arg); #ifdef Py_GIL_DISABLED extern void _PyGC_VisitObjectsWorldStopped(PyInterpreterState *interp, diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index c1fc3511f849ade..c86ae242feac1ed 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -22,7 +22,7 @@ PyGenObject *_PyGen_GetGeneratorFromFrame(_PyInterpreterFrame *frame) } PyAPI_FUNC(PyObject *)_PyGen_yf(PyGenObject *); -extern void _PyGen_Finalize(PyObject *self); +extern int _PyGen_ClearFrame(PyGenObject *self); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); @@ -31,12 +31,17 @@ PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); PyAPI_FUNC(PyObject *)_PyCoro_GetAwaitableIter(PyObject *o); -extern PyObject *_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); +PyAPI_FUNC(PyObject *)_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); + +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyCoro_ComputeOrigin(int origin_depth, _PyInterpreterFrame *current_frame); extern PyTypeObject _PyCoroWrapper_Type; extern PyTypeObject _PyAsyncGenWrappedValue_Type; extern PyTypeObject _PyAsyncGenAThrow_Type; +PyAPI_FUNC(PySendResult) _PyAsyncGenASend_Send(PyObject *iter, PyObject *arg, PyObject **result); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 493377b4c250407..1cf766ddb382dd2 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -13,7 +13,7 @@ static inline void _PyStaticObject_CheckRefcnt(PyObject *obj) { if (!_Py_IsImmortal(obj)) { fprintf(stderr, "Immortal Object has less refcnt than expected.\n"); - _PyObject_Dump(obj); + PyObject_Dump(obj); } } #endif @@ -286,6 +286,774 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 254]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 255]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 256]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 257]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 258]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 259]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 260]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 261]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 262]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 263]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 264]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 265]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 266]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 267]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 268]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 269]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 270]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 271]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 272]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 273]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 274]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 275]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 276]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 277]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 278]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 279]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 280]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 281]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 282]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 283]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 284]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 285]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 286]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 287]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 288]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 289]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 290]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 291]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 292]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 293]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 294]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 295]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 296]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 297]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 298]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 299]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 300]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 301]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 302]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 303]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 304]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 305]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 306]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 307]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 308]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 309]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 310]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 311]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 312]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 313]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 314]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 315]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 316]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 317]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 318]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 319]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 320]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 321]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 322]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 323]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 324]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 325]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 326]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 327]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 328]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 329]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 330]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 331]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 332]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 333]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 334]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 335]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 336]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 337]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 338]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 339]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 340]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 341]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 342]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 343]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 344]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 345]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 346]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 347]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 348]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 349]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 350]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 351]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 352]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 353]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 354]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 355]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 356]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 357]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 358]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 359]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 360]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 361]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 362]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 363]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 364]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 365]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 366]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 367]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 368]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 369]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 370]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 371]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 372]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 373]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 374]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 375]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 376]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 377]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 378]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 379]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 380]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 381]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 382]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 383]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 384]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 385]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 386]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 387]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 388]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 389]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 390]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 391]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 392]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 393]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 394]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 395]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 396]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 397]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 398]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 399]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 400]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 401]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 402]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 403]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 404]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 405]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 406]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 407]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 408]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 409]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 410]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 411]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 412]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 413]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 414]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 415]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 416]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 417]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 418]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 419]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 420]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 421]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 422]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 423]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 424]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 425]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 426]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 427]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 428]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 429]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 430]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 431]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 432]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 433]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 434]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 435]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 436]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 437]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 438]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 439]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 440]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 441]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 442]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 443]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 444]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 445]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 446]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 447]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 448]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 449]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 450]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 451]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 452]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 453]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 454]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 455]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 456]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 457]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 458]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 459]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 460]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 461]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 462]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 463]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 464]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 465]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 466]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 467]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 468]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 469]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 470]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 471]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 472]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 473]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 474]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 475]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 476]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 477]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 478]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 479]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 480]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 481]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 482]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 483]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 484]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 485]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 486]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 487]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 488]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 489]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 490]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 491]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 492]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 493]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 494]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 495]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 496]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 497]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 498]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 499]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 500]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 501]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 502]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 503]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 504]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 505]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 506]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 507]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 508]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 509]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 510]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 511]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 512]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 513]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 514]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 515]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 516]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 517]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 518]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 519]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 520]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 521]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 522]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 523]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 524]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 525]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 526]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 527]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 528]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 529]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 530]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 531]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 532]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 533]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 534]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 535]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 536]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 537]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 538]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 539]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 540]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 541]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 542]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 543]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 544]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 545]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 546]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 547]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 548]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 549]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 550]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 551]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 552]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 553]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 554]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 555]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 556]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 557]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 558]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 559]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 560]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 561]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 562]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 563]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 564]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 565]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 566]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 567]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 568]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 569]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 570]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 571]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 572]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 573]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 574]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 575]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 576]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 577]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 578]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 579]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 580]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 581]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 582]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 583]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 584]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 585]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 586]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 587]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 588]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 589]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 590]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 591]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 592]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 593]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 594]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 595]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 596]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 597]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 598]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 599]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 600]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 601]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 602]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 603]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 604]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 605]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 606]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 607]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 608]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 609]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 610]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 611]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 612]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 613]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 614]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 615]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 616]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 617]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 618]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 619]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 620]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 621]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 622]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 623]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 624]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 625]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 626]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 627]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 628]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 629]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 630]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 631]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 632]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 633]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 634]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 635]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 636]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 637]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 638]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 639]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 640]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 641]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 642]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 643]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 644]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 645]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 646]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 647]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 648]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 649]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 650]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 651]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 652]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 653]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 654]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 655]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 656]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 657]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 658]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 659]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 660]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 661]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 662]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 663]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 664]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 665]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 666]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 667]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 668]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 669]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 670]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 671]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 672]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 673]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 674]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 675]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 676]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 677]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 678]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 679]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 680]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 681]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 682]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 683]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 684]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 685]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 686]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 687]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 688]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 689]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 690]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 691]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 692]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 693]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 694]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 695]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 696]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 697]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 698]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 699]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 700]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 701]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 702]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 703]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 704]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 705]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 706]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 707]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 708]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 709]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 710]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 711]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 712]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 713]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 714]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 715]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 716]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 717]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 718]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 719]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 720]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 721]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 722]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 723]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 724]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 725]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 726]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 727]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 728]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 729]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 730]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 731]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 732]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 733]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 734]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 735]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 736]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 737]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 738]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 739]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 740]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 741]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 742]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 743]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 744]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 745]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 746]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 747]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 748]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 749]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 750]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 751]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 752]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 753]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 754]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 755]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 756]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 757]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 758]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 759]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 760]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 761]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 762]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 763]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 764]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 765]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 766]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 767]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 768]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 769]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 770]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 771]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 772]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 773]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 774]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 775]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 776]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 777]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 778]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 779]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 780]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 781]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 782]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 783]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 784]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 785]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 786]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 787]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 788]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 789]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 790]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 791]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 792]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 793]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 794]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 795]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 796]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 797]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 798]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 799]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 800]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 801]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 802]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 803]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 804]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 805]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 806]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 807]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 808]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 809]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 810]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 811]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 812]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 813]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 814]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 815]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 816]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 817]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 818]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 819]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 820]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 821]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 822]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 823]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 824]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 825]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 826]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 827]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 828]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 829]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 830]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 831]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 832]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 833]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 834]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 835]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 836]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 837]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 838]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 839]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 840]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 841]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 842]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 843]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 844]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 845]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 846]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 847]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 848]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 849]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 850]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 851]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 852]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 853]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 854]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 855]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 856]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 857]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 858]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 859]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 860]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 861]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 862]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 863]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 864]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 865]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 866]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 867]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 868]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 869]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 870]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 871]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 872]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 873]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 874]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 875]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 876]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 877]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 878]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 879]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 880]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 881]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 882]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 883]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 884]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 885]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 886]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 887]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 888]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 889]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 890]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 891]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 892]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 893]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 894]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 895]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 896]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 897]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 898]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 899]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 900]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 901]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 902]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 903]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 904]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 905]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 906]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 907]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 908]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 909]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 910]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 911]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 912]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 913]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 914]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 915]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 916]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 917]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 918]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 919]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 920]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 921]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 922]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 923]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 924]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 925]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 926]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 927]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 928]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 929]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 930]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 931]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 932]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 933]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 934]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 935]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 936]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 937]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 938]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 939]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 940]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 941]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 942]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 943]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 944]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 945]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 946]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 947]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 948]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 949]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 950]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 951]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 952]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 953]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 954]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 955]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 956]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 957]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 958]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 959]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 960]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 961]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 962]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 963]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 964]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 965]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 966]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 967]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 968]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 969]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 970]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 971]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 972]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 973]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 974]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 975]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 976]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 977]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 978]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 979]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 980]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 981]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 982]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 983]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 984]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 985]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 986]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 987]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 988]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 989]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 990]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 991]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 992]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 993]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 994]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 995]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 996]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 997]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 998]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 999]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1000]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1001]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1002]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1003]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1004]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1005]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1006]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1007]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1008]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1009]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1010]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1011]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1012]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1013]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1014]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1015]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1016]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1017]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1018]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1019]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1020]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1021]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1022]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1023]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1024]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(bytes_characters)[0]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(bytes_characters)[1]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(bytes_characters)[2]); @@ -558,16 +1326,32 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot_locals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(empty)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(format)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(gc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(generic_base)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(list_err)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(native)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(str_replace_inf)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(AGEN_CLOSED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(AGEN_CREATED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(AGEN_RUNNING)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(AGEN_SUSPENDED)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CANCELLED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CORO_CLOSED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CORO_CREATED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CORO_RUNNING)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CORO_SUSPENDED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(Emax)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(Emin)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(FINISHED)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(False)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(GEN_CLOSED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(GEN_CREATED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(GEN_RUNNING)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(GEN_SUSPENDED)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(JSONDecodeError)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(PENDING)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(Py_Repr)); @@ -662,6 +1446,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__iter__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__itruediv__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__ixor__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__lazy_import__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__lazy_modules__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__le__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__len__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__length_hint__)); @@ -790,13 +1576,16 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aclose)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(add)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(add_done_callback)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(adobe)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_child)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(align)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all_interpreters)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alphabet)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(any)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arg)); @@ -818,6 +1607,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bit_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bit_size)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(block)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(blocking)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bound)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(buffer)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(buffer_callback)); @@ -835,6 +1625,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_exception)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_parameter_type)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_return)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_stack)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cache_frames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_datetime_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_statements)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cadata)); @@ -842,13 +1634,18 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(call)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(call_exception_handler)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(call_soon)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(callable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cancel)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(canonical)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capath)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capitals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(category)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cb_type)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(certfile)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(chain)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(check_same_thread)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(clamp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(clear)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(close)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(closed)); @@ -874,9 +1671,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(co_varnames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(col_offset)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(collector)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(command)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(comment_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compile_mode)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compression)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(config)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(consts)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(context)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(contravariant)); @@ -887,11 +1687,14 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(coro)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(count)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(covariant)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ctx)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cwd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(d_parameter_type)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(data)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(database)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(date)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(day)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(days)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(debug)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decoder)); @@ -921,6 +1724,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eager_start)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(effective_ids)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(element_factory)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(emptyerror)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(encode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(encoding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end)); @@ -933,17 +1737,22 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(errors)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(event)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eventmask)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_tb)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_type)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_val)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(excepthook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exception)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(existing_file_name)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(expression)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extend)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extra_tokens)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(facility)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(factory)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fallback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(false)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(family)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fanout)); @@ -966,17 +1775,21 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(flags)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(flush)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fold)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(foldspaces)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(follow_symlinks)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format_spec)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(frame_buffer)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(free_threaded)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(from_param)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromlist)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromtimestamp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromutc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fset)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fullerror)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(func)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(future)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(gc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(generation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get_debug)); @@ -998,13 +1811,17 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hi)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hour)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hours)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(id)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ignore)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ignorechars)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(imag)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(implieslink)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(importlib)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(in_fd)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(include_aliases)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(incoming)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(index)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(indexgroup)); @@ -1051,6 +1868,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw2)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kwargs)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(label)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last)); @@ -1061,6 +1879,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(latin1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(leaf_size)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(legacy)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(len)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(length)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(level)); @@ -1076,12 +1895,15 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(loop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(manual_reset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mapping)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mask)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(match)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_length)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxdigits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxevents)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxlen)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxmem)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxsize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxsplit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxvalue)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(memLevel)); @@ -1091,13 +1913,16 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(method)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(microsecond)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(microseconds)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(milliseconds)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minute)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minutes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mod)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module_globals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(modules)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(modulo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(month)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mro)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(msg)); @@ -1110,6 +1935,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(name_from)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespace_separator)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespaces)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(native)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ndigits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nested)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_file_name)); @@ -1136,6 +1962,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(only_keys)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(oparg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(opcode)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(opcodes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(open)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(opener)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(operation)); @@ -1143,11 +1970,14 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(options)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(order)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(origin)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(other)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(out_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outgoing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outpath)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(overlapped)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(owner)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pad)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(padded)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pages)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(parameter)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(parent)); @@ -1160,20 +1990,28 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(person)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pi_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pid)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pidfd)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pointer_bits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(policy)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos2)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(posix)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(prec)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(preserve_exc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(print_file_and_line)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress_callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress_routine)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(proto)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(protocol)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ps1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ps2)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(qid)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(qualname)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(query)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(queuetype)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(quotetabs)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(raw)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(read)); @@ -1185,24 +2023,33 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(readline)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(readonly)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(real)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(recursive)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reducer_override)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(registry)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(rel_tol)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(release)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reload)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repeat)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repl)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(replace)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repr)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reqrefs)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(require_ready)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reserved)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(resetids)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(restrict)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(return)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reverse)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reversed)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(rounding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(salt)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sample_interval_us)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sched_priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(scheduler)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(script)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(second)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seconds)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(security_attributes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seek)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seekable)); @@ -1220,11 +2067,15 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(setsigmask)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(setstate)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(shape)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(shared)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(short)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(show_cmd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(signed)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(signum)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(size)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sizehint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(skip_file_prefixes)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(skip_non_matching_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sleep)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sock)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sort)); @@ -1233,9 +2084,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(spam)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(src)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(src_dir_fd)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stack_frames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stacklevel)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(start)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(start_time_us)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(statement)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stats)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(status)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stderr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stdin)); @@ -1253,8 +2107,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(symmetric_difference_update)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tabsize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tag)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(take_bytes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(target)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(target_is_directory)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(targetfd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(task)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tb_frame)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tb_lasti)); @@ -1264,17 +2120,23 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(template)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(term)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(text)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(third)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(threading)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(throw)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(time)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timeout)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timer)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(times)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timespec)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timestamp)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timestamp_us)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timetuple)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timeunit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(top)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(traceback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trailers)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(translate)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(traps)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(true)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(truncate)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(twice)); @@ -1285,10 +2147,13 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tzinfo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tzname)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(uid)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(unboundop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(unlink)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(unraisablehook)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(updates)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(uri)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(usedforsecurity)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(utcoffset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(values)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(version)); @@ -1300,9 +2165,11 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wbits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(week)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(weekday)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(weeks)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(which)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(who)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(withdata)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wrapcol)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(writable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(write)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(write_through)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 5dfea2f479d5fec..017d62e002fdff9 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -46,19 +46,35 @@ struct _Py_global_strings { STRUCT_FOR_STR(dot_locals, ".") STRUCT_FOR_STR(empty, "") STRUCT_FOR_STR(format, ".format") + STRUCT_FOR_STR(gc, "") STRUCT_FOR_STR(generic_base, ".generic_base") STRUCT_FOR_STR(json_decoder, "json.decoder") STRUCT_FOR_STR(kwdefaults, ".kwdefaults") STRUCT_FOR_STR(list_err, "list index out of range") + STRUCT_FOR_STR(native, "") STRUCT_FOR_STR(str_replace_inf, "1e309") STRUCT_FOR_STR(type_params, ".type_params") STRUCT_FOR_STR(utf_8, "utf-8") } literals; struct { + STRUCT_FOR_ID(AGEN_CLOSED) + STRUCT_FOR_ID(AGEN_CREATED) + STRUCT_FOR_ID(AGEN_RUNNING) + STRUCT_FOR_ID(AGEN_SUSPENDED) STRUCT_FOR_ID(CANCELLED) + STRUCT_FOR_ID(CORO_CLOSED) + STRUCT_FOR_ID(CORO_CREATED) + STRUCT_FOR_ID(CORO_RUNNING) + STRUCT_FOR_ID(CORO_SUSPENDED) + STRUCT_FOR_ID(Emax) + STRUCT_FOR_ID(Emin) STRUCT_FOR_ID(FINISHED) STRUCT_FOR_ID(False) + STRUCT_FOR_ID(GEN_CLOSED) + STRUCT_FOR_ID(GEN_CREATED) + STRUCT_FOR_ID(GEN_RUNNING) + STRUCT_FOR_ID(GEN_SUSPENDED) STRUCT_FOR_ID(JSONDecodeError) STRUCT_FOR_ID(PENDING) STRUCT_FOR_ID(Py_Repr) @@ -153,6 +169,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(__iter__) STRUCT_FOR_ID(__itruediv__) STRUCT_FOR_ID(__ixor__) + STRUCT_FOR_ID(__lazy_import__) + STRUCT_FOR_ID(__lazy_modules__) STRUCT_FOR_ID(__le__) STRUCT_FOR_ID(__len__) STRUCT_FOR_ID(__length_hint__) @@ -281,13 +299,16 @@ struct _Py_global_strings { STRUCT_FOR_ID(aclose) STRUCT_FOR_ID(add) STRUCT_FOR_ID(add_done_callback) + STRUCT_FOR_ID(adobe) STRUCT_FOR_ID(after_in_child) STRUCT_FOR_ID(after_in_parent) STRUCT_FOR_ID(alias) STRUCT_FOR_ID(align) STRUCT_FOR_ID(all) + STRUCT_FOR_ID(all_interpreters) STRUCT_FOR_ID(all_threads) STRUCT_FOR_ID(allow_code) + STRUCT_FOR_ID(alphabet) STRUCT_FOR_ID(any) STRUCT_FOR_ID(append) STRUCT_FOR_ID(arg) @@ -309,6 +330,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(bit_offset) STRUCT_FOR_ID(bit_size) STRUCT_FOR_ID(block) + STRUCT_FOR_ID(blocking) STRUCT_FOR_ID(bound) STRUCT_FOR_ID(buffer) STRUCT_FOR_ID(buffer_callback) @@ -326,6 +348,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(c_exception) STRUCT_FOR_ID(c_parameter_type) STRUCT_FOR_ID(c_return) + STRUCT_FOR_ID(c_stack) + STRUCT_FOR_ID(cache_frames) STRUCT_FOR_ID(cached_datetime_module) STRUCT_FOR_ID(cached_statements) STRUCT_FOR_ID(cadata) @@ -333,13 +357,18 @@ struct _Py_global_strings { STRUCT_FOR_ID(call) STRUCT_FOR_ID(call_exception_handler) STRUCT_FOR_ID(call_soon) + STRUCT_FOR_ID(callable) STRUCT_FOR_ID(callback) STRUCT_FOR_ID(cancel) + STRUCT_FOR_ID(canonical) STRUCT_FOR_ID(capath) + STRUCT_FOR_ID(capitals) STRUCT_FOR_ID(category) STRUCT_FOR_ID(cb_type) STRUCT_FOR_ID(certfile) + STRUCT_FOR_ID(chain) STRUCT_FOR_ID(check_same_thread) + STRUCT_FOR_ID(clamp) STRUCT_FOR_ID(clear) STRUCT_FOR_ID(close) STRUCT_FOR_ID(closed) @@ -365,9 +394,12 @@ struct _Py_global_strings { STRUCT_FOR_ID(co_varnames) STRUCT_FOR_ID(code) STRUCT_FOR_ID(col_offset) + STRUCT_FOR_ID(collector) STRUCT_FOR_ID(command) STRUCT_FOR_ID(comment_factory) STRUCT_FOR_ID(compile_mode) + STRUCT_FOR_ID(compression) + STRUCT_FOR_ID(config) STRUCT_FOR_ID(consts) STRUCT_FOR_ID(context) STRUCT_FOR_ID(contravariant) @@ -378,11 +410,14 @@ struct _Py_global_strings { STRUCT_FOR_ID(coro) STRUCT_FOR_ID(count) STRUCT_FOR_ID(covariant) + STRUCT_FOR_ID(ctx) STRUCT_FOR_ID(cwd) STRUCT_FOR_ID(d_parameter_type) STRUCT_FOR_ID(data) STRUCT_FOR_ID(database) + STRUCT_FOR_ID(date) STRUCT_FOR_ID(day) + STRUCT_FOR_ID(days) STRUCT_FOR_ID(debug) STRUCT_FOR_ID(decode) STRUCT_FOR_ID(decoder) @@ -412,6 +447,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(eager_start) STRUCT_FOR_ID(effective_ids) STRUCT_FOR_ID(element_factory) + STRUCT_FOR_ID(emptyerror) STRUCT_FOR_ID(encode) STRUCT_FOR_ID(encoding) STRUCT_FOR_ID(end) @@ -424,17 +460,22 @@ struct _Py_global_strings { STRUCT_FOR_ID(errors) STRUCT_FOR_ID(event) STRUCT_FOR_ID(eventmask) + STRUCT_FOR_ID(exc) + STRUCT_FOR_ID(exc_tb) STRUCT_FOR_ID(exc_type) + STRUCT_FOR_ID(exc_val) STRUCT_FOR_ID(exc_value) STRUCT_FOR_ID(excepthook) STRUCT_FOR_ID(exception) STRUCT_FOR_ID(existing_file_name) + STRUCT_FOR_ID(exit) STRUCT_FOR_ID(exp) STRUCT_FOR_ID(expression) STRUCT_FOR_ID(extend) STRUCT_FOR_ID(extra_tokens) STRUCT_FOR_ID(facility) STRUCT_FOR_ID(factory) + STRUCT_FOR_ID(fallback) STRUCT_FOR_ID(false) STRUCT_FOR_ID(family) STRUCT_FOR_ID(fanout) @@ -457,17 +498,21 @@ struct _Py_global_strings { STRUCT_FOR_ID(flags) STRUCT_FOR_ID(flush) STRUCT_FOR_ID(fold) + STRUCT_FOR_ID(foldspaces) STRUCT_FOR_ID(follow_symlinks) STRUCT_FOR_ID(format) STRUCT_FOR_ID(format_spec) STRUCT_FOR_ID(frame_buffer) + STRUCT_FOR_ID(free_threaded) STRUCT_FOR_ID(from_param) STRUCT_FOR_ID(fromlist) STRUCT_FOR_ID(fromtimestamp) STRUCT_FOR_ID(fromutc) STRUCT_FOR_ID(fset) + STRUCT_FOR_ID(fullerror) STRUCT_FOR_ID(func) STRUCT_FOR_ID(future) + STRUCT_FOR_ID(gc) STRUCT_FOR_ID(generation) STRUCT_FOR_ID(get) STRUCT_FOR_ID(get_debug) @@ -489,13 +534,17 @@ struct _Py_global_strings { STRUCT_FOR_ID(hi) STRUCT_FOR_ID(hook) STRUCT_FOR_ID(hour) + STRUCT_FOR_ID(hours) STRUCT_FOR_ID(id) STRUCT_FOR_ID(ident) STRUCT_FOR_ID(identity_hint) STRUCT_FOR_ID(ignore) + STRUCT_FOR_ID(ignorechars) STRUCT_FOR_ID(imag) + STRUCT_FOR_ID(implieslink) STRUCT_FOR_ID(importlib) STRUCT_FOR_ID(in_fd) + STRUCT_FOR_ID(include_aliases) STRUCT_FOR_ID(incoming) STRUCT_FOR_ID(index) STRUCT_FOR_ID(indexgroup) @@ -542,6 +591,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(kw) STRUCT_FOR_ID(kw1) STRUCT_FOR_ID(kw2) + STRUCT_FOR_ID(kwargs) STRUCT_FOR_ID(kwdefaults) STRUCT_FOR_ID(label) STRUCT_FOR_ID(last) @@ -552,6 +602,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(last_value) STRUCT_FOR_ID(latin1) STRUCT_FOR_ID(leaf_size) + STRUCT_FOR_ID(legacy) STRUCT_FOR_ID(len) STRUCT_FOR_ID(length) STRUCT_FOR_ID(level) @@ -567,12 +618,15 @@ struct _Py_global_strings { STRUCT_FOR_ID(loop) STRUCT_FOR_ID(manual_reset) STRUCT_FOR_ID(mapping) + STRUCT_FOR_ID(mask) STRUCT_FOR_ID(match) STRUCT_FOR_ID(max_length) + STRUCT_FOR_ID(max_threads) STRUCT_FOR_ID(maxdigits) STRUCT_FOR_ID(maxevents) STRUCT_FOR_ID(maxlen) STRUCT_FOR_ID(maxmem) + STRUCT_FOR_ID(maxsize) STRUCT_FOR_ID(maxsplit) STRUCT_FOR_ID(maxvalue) STRUCT_FOR_ID(memLevel) @@ -582,13 +636,16 @@ struct _Py_global_strings { STRUCT_FOR_ID(metadata) STRUCT_FOR_ID(method) STRUCT_FOR_ID(microsecond) + STRUCT_FOR_ID(microseconds) STRUCT_FOR_ID(milliseconds) STRUCT_FOR_ID(minute) + STRUCT_FOR_ID(minutes) STRUCT_FOR_ID(mod) STRUCT_FOR_ID(mode) STRUCT_FOR_ID(module) STRUCT_FOR_ID(module_globals) STRUCT_FOR_ID(modules) + STRUCT_FOR_ID(modulo) STRUCT_FOR_ID(month) STRUCT_FOR_ID(mro) STRUCT_FOR_ID(msg) @@ -601,6 +658,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(name_from) STRUCT_FOR_ID(namespace_separator) STRUCT_FOR_ID(namespaces) + STRUCT_FOR_ID(native) STRUCT_FOR_ID(ndigits) STRUCT_FOR_ID(nested) STRUCT_FOR_ID(new_file_name) @@ -627,6 +685,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(only_keys) STRUCT_FOR_ID(oparg) STRUCT_FOR_ID(opcode) + STRUCT_FOR_ID(opcodes) STRUCT_FOR_ID(open) STRUCT_FOR_ID(opener) STRUCT_FOR_ID(operation) @@ -634,11 +693,14 @@ struct _Py_global_strings { STRUCT_FOR_ID(options) STRUCT_FOR_ID(order) STRUCT_FOR_ID(origin) + STRUCT_FOR_ID(other) STRUCT_FOR_ID(out_fd) STRUCT_FOR_ID(outgoing) STRUCT_FOR_ID(outpath) STRUCT_FOR_ID(overlapped) STRUCT_FOR_ID(owner) + STRUCT_FOR_ID(pad) + STRUCT_FOR_ID(padded) STRUCT_FOR_ID(pages) STRUCT_FOR_ID(parameter) STRUCT_FOR_ID(parent) @@ -651,20 +713,28 @@ struct _Py_global_strings { STRUCT_FOR_ID(person) STRUCT_FOR_ID(pi_factory) STRUCT_FOR_ID(pid) + STRUCT_FOR_ID(pidfd) + STRUCT_FOR_ID(pointer_bits) STRUCT_FOR_ID(policy) STRUCT_FOR_ID(pos) STRUCT_FOR_ID(pos1) STRUCT_FOR_ID(pos2) STRUCT_FOR_ID(posix) + STRUCT_FOR_ID(prec) + STRUCT_FOR_ID(preserve_exc) STRUCT_FOR_ID(print_file_and_line) STRUCT_FOR_ID(priority) STRUCT_FOR_ID(progress) + STRUCT_FOR_ID(progress_callback) STRUCT_FOR_ID(progress_routine) STRUCT_FOR_ID(proto) STRUCT_FOR_ID(protocol) STRUCT_FOR_ID(ps1) STRUCT_FOR_ID(ps2) + STRUCT_FOR_ID(qid) + STRUCT_FOR_ID(qualname) STRUCT_FOR_ID(query) + STRUCT_FOR_ID(queuetype) STRUCT_FOR_ID(quotetabs) STRUCT_FOR_ID(raw) STRUCT_FOR_ID(read) @@ -676,24 +746,33 @@ struct _Py_global_strings { STRUCT_FOR_ID(readline) STRUCT_FOR_ID(readonly) STRUCT_FOR_ID(real) + STRUCT_FOR_ID(recursive) STRUCT_FOR_ID(reducer_override) STRUCT_FOR_ID(registry) STRUCT_FOR_ID(rel_tol) STRUCT_FOR_ID(release) STRUCT_FOR_ID(reload) + STRUCT_FOR_ID(repeat) STRUCT_FOR_ID(repl) STRUCT_FOR_ID(replace) + STRUCT_FOR_ID(repr) + STRUCT_FOR_ID(reqrefs) + STRUCT_FOR_ID(require_ready) STRUCT_FOR_ID(reserved) STRUCT_FOR_ID(reset) STRUCT_FOR_ID(resetids) + STRUCT_FOR_ID(restrict) STRUCT_FOR_ID(return) STRUCT_FOR_ID(reverse) STRUCT_FOR_ID(reversed) + STRUCT_FOR_ID(rounding) STRUCT_FOR_ID(salt) + STRUCT_FOR_ID(sample_interval_us) STRUCT_FOR_ID(sched_priority) STRUCT_FOR_ID(scheduler) STRUCT_FOR_ID(script) STRUCT_FOR_ID(second) + STRUCT_FOR_ID(seconds) STRUCT_FOR_ID(security_attributes) STRUCT_FOR_ID(seek) STRUCT_FOR_ID(seekable) @@ -711,11 +790,15 @@ struct _Py_global_strings { STRUCT_FOR_ID(setsigmask) STRUCT_FOR_ID(setstate) STRUCT_FOR_ID(shape) + STRUCT_FOR_ID(shared) + STRUCT_FOR_ID(short) STRUCT_FOR_ID(show_cmd) STRUCT_FOR_ID(signed) + STRUCT_FOR_ID(signum) STRUCT_FOR_ID(size) STRUCT_FOR_ID(sizehint) STRUCT_FOR_ID(skip_file_prefixes) + STRUCT_FOR_ID(skip_non_matching_threads) STRUCT_FOR_ID(sleep) STRUCT_FOR_ID(sock) STRUCT_FOR_ID(sort) @@ -724,9 +807,12 @@ struct _Py_global_strings { STRUCT_FOR_ID(spam) STRUCT_FOR_ID(src) STRUCT_FOR_ID(src_dir_fd) + STRUCT_FOR_ID(stack_frames) STRUCT_FOR_ID(stacklevel) STRUCT_FOR_ID(start) + STRUCT_FOR_ID(start_time_us) STRUCT_FOR_ID(statement) + STRUCT_FOR_ID(stats) STRUCT_FOR_ID(status) STRUCT_FOR_ID(stderr) STRUCT_FOR_ID(stdin) @@ -744,8 +830,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(symmetric_difference_update) STRUCT_FOR_ID(tabsize) STRUCT_FOR_ID(tag) + STRUCT_FOR_ID(take_bytes) STRUCT_FOR_ID(target) STRUCT_FOR_ID(target_is_directory) + STRUCT_FOR_ID(targetfd) STRUCT_FOR_ID(task) STRUCT_FOR_ID(tb_frame) STRUCT_FOR_ID(tb_lasti) @@ -755,17 +843,23 @@ struct _Py_global_strings { STRUCT_FOR_ID(template) STRUCT_FOR_ID(term) STRUCT_FOR_ID(text) + STRUCT_FOR_ID(third) STRUCT_FOR_ID(threading) STRUCT_FOR_ID(throw) + STRUCT_FOR_ID(time) STRUCT_FOR_ID(timeout) STRUCT_FOR_ID(timer) STRUCT_FOR_ID(times) + STRUCT_FOR_ID(timespec) + STRUCT_FOR_ID(timestamp) + STRUCT_FOR_ID(timestamp_us) STRUCT_FOR_ID(timetuple) STRUCT_FOR_ID(timeunit) STRUCT_FOR_ID(top) STRUCT_FOR_ID(traceback) STRUCT_FOR_ID(trailers) STRUCT_FOR_ID(translate) + STRUCT_FOR_ID(traps) STRUCT_FOR_ID(true) STRUCT_FOR_ID(truncate) STRUCT_FOR_ID(twice) @@ -776,10 +870,13 @@ struct _Py_global_strings { STRUCT_FOR_ID(tzinfo) STRUCT_FOR_ID(tzname) STRUCT_FOR_ID(uid) + STRUCT_FOR_ID(unboundop) STRUCT_FOR_ID(unlink) STRUCT_FOR_ID(unraisablehook) + STRUCT_FOR_ID(updates) STRUCT_FOR_ID(uri) STRUCT_FOR_ID(usedforsecurity) + STRUCT_FOR_ID(utcoffset) STRUCT_FOR_ID(value) STRUCT_FOR_ID(values) STRUCT_FOR_ID(version) @@ -791,9 +888,11 @@ struct _Py_global_strings { STRUCT_FOR_ID(wbits) STRUCT_FOR_ID(week) STRUCT_FOR_ID(weekday) + STRUCT_FOR_ID(weeks) STRUCT_FOR_ID(which) STRUCT_FOR_ID(who) STRUCT_FOR_ID(withdata) + STRUCT_FOR_ID(wrapcol) STRUCT_FOR_ID(writable) STRUCT_FOR_ID(write) STRUCT_FOR_ID(write_through) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 13fbff4eb65cb2a..a1078828afa572e 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -17,7 +17,8 @@ extern int _PyImport_IsInitialized(PyInterpreterState *); // Export for 'pyexpat' shared extension PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); -extern int _PyImport_SetModuleString(const char *name, PyObject* module); +// Export for 'math' shared extension +PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module); extern void _PyImport_AcquireLock(PyInterpreterState *interp); extern void _PyImport_ReleaseLock(PyInterpreterState *interp); @@ -31,6 +32,20 @@ extern int _PyImport_FixupBuiltin( PyObject *modules ); +extern PyObject * _PyImport_ResolveName( + PyThreadState *tstate, PyObject *name, PyObject *globals, int level); +extern PyObject * _PyImport_GetAbsName( + PyThreadState *tstate, PyObject *name, PyObject *globals, int level); +// Symbol is exported for the JIT on Windows builds. +PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate( + PyThreadState *tstate, PyObject *lazy_import); +extern PyObject * _PyImport_TryLoadLazySubmodule( + PyObject *mod_name, PyObject *attr_name); +extern PyObject * _PyImport_LazyImportModuleLevelObject( + PyThreadState *tstate, PyObject *name, PyObject *builtins, + PyObject *globals, PyObject *locals, PyObject *fromlist, int level); + + #ifdef HAVE_DLOPEN # include // RTLD_NOW, RTLD_LAZY # if HAVE_DECL_RTLD_NOW @@ -68,11 +83,19 @@ extern void _PyImport_ClearModules(PyInterpreterState *interp); extern void _PyImport_ClearModulesByIndex(PyInterpreterState *interp); +extern PyObject * _PyImport_InitLazyModules( + PyInterpreterState *interp); +extern void _PyImport_ClearLazyModules(PyInterpreterState *interp); + extern int _PyImport_InitDefaultImportFunc(PyInterpreterState *interp); extern int _PyImport_IsDefaultImportFunc( PyInterpreterState *interp, PyObject *func); +extern int _PyImport_IsDefaultLazyImportFunc( + PyInterpreterState *interp, + PyObject *func); + extern PyObject * _PyImport_GetImportlibLoader( PyInterpreterState *interp, const char *loader_name); @@ -127,11 +150,18 @@ PyAPI_FUNC(int) _PyImport_ClearExtension(PyObject *name, PyObject *filename); // state of the module argument: // - If module is NULL or a PyModuleObject with md_gil == Py_MOD_GIL_NOT_USED, // call _PyEval_DisableGIL(). -// - Otherwise, call _PyEval_EnableGILPermanent(). If the GIL was not already -// enabled permanently, issue a warning referencing the module's name. +// - Otherwise, call _PyImport_EnableGILAndWarn // // This function may raise an exception. extern int _PyImport_CheckGILForModule(PyObject *module, PyObject *module_name); +// Assuming that the GIL is enabled from a call to +// _PyEval_EnableGILTransient(), call _PyEval_EnableGILPermanent(). +// If the GIL was not already enabled permanently, issue a warning referencing +// the module's name. +// Leave a message in verbose mode. +// +// This function may raise an exception. +extern int _PyImport_EnableGILAndWarn(PyThreadState *, PyObject *module_name); #endif #ifdef __cplusplus diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index 3ba9229cc213787..9ed87a544234c59 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -14,6 +14,34 @@ extern "C" { extern const char *_PyImport_DynLoadFiletab[]; +#ifdef HAVE_DYNAMIC_LOADING +/* ./configure sets HAVE_DYNAMIC_LOADING if dynamic loading of modules is + supported on this platform. configure will then compile and link in one + of the dynload_*.c files, as appropriate. We will call a function in + those modules to get a function pointer to the module's init function. + + The function should return: + - The function pointer on success + - NULL with exception set if the library cannot be loaded + - NULL *without* an extension set if the library could be loaded but the + function cannot be found in it. +*/ +#ifdef MS_WINDOWS +#include +typedef FARPROC dl_funcptr; +extern dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, + const char *shortname, + PyObject *pathname, + FILE *fp); +#else +typedef void (*dl_funcptr)(void); +extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, + const char *shortname, + const char *pathname, FILE *fp); +#endif + +#endif /* HAVE_DYNAMIC_LOADING */ + typedef enum ext_module_kind { _Py_ext_module_kind_UNKNOWN = 0, @@ -28,6 +56,11 @@ typedef enum ext_module_origin { _Py_ext_module_origin_DYNAMIC = 3, } _Py_ext_module_origin; +struct hook_prefixes { + const char *const init_prefix; + const char *const export_prefix; +}; + /* Input for loading an extension module. */ struct _Py_ext_module_loader_info { PyObject *filename; @@ -40,7 +73,7 @@ struct _Py_ext_module_loader_info { * depending on if it's builtin or not. */ PyObject *path; _Py_ext_module_origin origin; - const char *hook_prefix; + const struct hook_prefixes *hook_prefixes; const char *newcontext; }; extern void _Py_ext_module_loader_info_clear( @@ -62,7 +95,9 @@ extern int _Py_ext_module_loader_info_init_from_spec( PyObject *spec); #endif -/* The result from running an extension module's init function. */ +/* The result from running an extension module's init function. + * Not used for modules defined via PyModExport (slots array). + */ struct _Py_ext_module_loader_result { PyModuleDef *def; PyObject *module; @@ -89,10 +124,11 @@ extern void _Py_ext_module_loader_result_apply_error( /* The module init function. */ typedef PyObject *(*PyModInitFunction)(void); +typedef PySlot *(*PyModExportFunction)(void); #ifdef HAVE_DYNAMIC_LOADING -extern PyModInitFunction _PyImport_GetModInitFunc( +extern int _PyImport_GetModuleExportHooks( struct _Py_ext_module_loader_info *info, - FILE *fp); + FILE *fp, PyModInitFunction *modinit, PyModExportFunction *modexport); #endif extern int _PyImport_RunModInitFunc( PyModInitFunction p0, @@ -104,8 +140,6 @@ extern int _PyImport_RunModInitFunc( #define MAXSUFFIXSIZE 12 #ifdef MS_WINDOWS -#include -typedef FARPROC dl_funcptr; #ifdef Py_DEBUG # define PYD_DEBUG_SUFFIX "_d" @@ -128,8 +162,6 @@ typedef FARPROC dl_funcptr; #define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX "." PYD_SOABI ".pyd" #define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" -#else -typedef void (*dl_funcptr)(void); #endif diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 368dafb90635d7b..183b2d45c5ede1c 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -153,10 +153,8 @@ typedef enum { } _PyConfigInitEnum; typedef enum { - /* For now, this means the GIL is enabled. - - gh-116329: This will eventually change to "the GIL is disabled but can - be re-enabled by loading an incompatible extension module." */ + /* In free threaded builds, this means that the GIL is disabled at startup, + but may be enabled by loading an incompatible extension module. */ _PyConfig_GIL_DEFAULT = -1, /* The GIL has been forced off or on, and will not be affected by module loading. */ diff --git a/Include/internal/pycore_instruction_sequence.h b/Include/internal/pycore_instruction_sequence.h index b5c927735374beb..6257b771fd9539c 100644 --- a/Include/internal/pycore_instruction_sequence.h +++ b/Include/internal/pycore_instruction_sequence.h @@ -73,6 +73,7 @@ int _PyInstructionSequence_SetAnnotationsCode(_PyInstructionSequence *seq, _PyInstructionSequence *annotations); int _PyInstructionSequence_AddNested(_PyInstructionSequence *seq, _PyInstructionSequence *nested); void PyInstructionSequence_Fini(_PyInstructionSequence *seq); +_PyInstruction _PyInstructionSequence_GetInstruction(_PyInstructionSequence *seq, int pos); extern PyTypeObject _PyInstructionSequence_Type; #define _PyInstructionSequence_Check(v) Py_IS_TYPE((v), &_PyInstructionSequence_Type) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 7658adca719e86d..56b55e93a014cb5 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -33,54 +33,52 @@ int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events); int _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet events); int _PyMonitoring_GetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet *events); -extern int + +// these are exported only for other re-generated interpreters to call +PyAPI_FUNC(int) _Py_call_instrumentation(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); -extern int +PyAPI_FUNC(int) _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev); -extern int +PyAPI_FUNC(int) _Py_call_instrumentation_instruction( PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr); -_Py_CODEUNIT * +PyAPI_FUNC(_Py_CODEUNIT *) _Py_call_instrumentation_jump( _Py_CODEUNIT *instr, PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest); -extern int +PyAPI_FUNC(int) _Py_call_instrumentation_arg(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); -extern int +PyAPI_FUNC(int) _Py_call_instrumentation_2args(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); -extern void +PyAPI_FUNC(void) _Py_call_instrumentation_exc2(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); -extern int -_Py_Instrumentation_GetLine(PyCodeObject *code, int index); - -extern PyObject _PyInstrumentation_MISSING; -extern PyObject _PyInstrumentation_DISABLE; +PyAPI_DATA(PyObject) _PyInstrumentation_MISSING; +PyAPI_DATA(PyObject) _PyInstrumentation_DISABLE; /* Total tool ids available */ #define PY_MONITORING_TOOL_IDS 8 -/* Count of all local monitoring events */ -#define _PY_MONITORING_LOCAL_EVENTS 11 -/* Count of all "real" monitoring events (not derived from other events) */ +/* Count of all "real" monitoring events (not derived from other events). + * "Other" events can now be turned on/disabled per code object. */ #define _PY_MONITORING_UNGROUPED_EVENTS 16 /* Count of all monitoring events */ #define _PY_MONITORING_EVENTS 19 /* Tables of which tools are active for each monitored event. */ typedef struct _Py_LocalMonitors { - uint8_t tools[_PY_MONITORING_LOCAL_EVENTS]; + uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS]; } _Py_LocalMonitors; typedef struct _Py_GlobalMonitors { @@ -120,6 +118,17 @@ typedef struct _PyCoMonitoringData { uint8_t *per_instruction_tools; } _PyCoMonitoringData; +extern int +_Py_Instrumentation_GetLine(PyCodeObject *code, _PyCoLineInstrumentationData *line_data, int index); + +static inline uint8_t +_PyCode_GetOriginalOpcode(_PyCoLineInstrumentationData *line_data, int index) +{ + return line_data->data[index*line_data->bytes_per_entry]; +} + +// Exported for external JIT support +PyAPI_FUNC(uint8_t) _PyCode_Deinstrument(uint8_t opcode); #ifdef __cplusplus } diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 542a75617b4d3cf..5500c70a3b0aad9 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -14,7 +14,7 @@ extern "C" { #include "pycore_structs.h" // PyHamtObject #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_typedefs.h" // _PyRuntimeState - +#include "pycore_uop.h" // _PyBloomFilter #define CODE_MAX_WATCHERS 8 #define CONTEXT_MAX_WATCHERS 8 @@ -69,7 +69,7 @@ struct code_arena_st; struct trampoline_api_st { void* (*init_state)(void); void (*write_state)(void* state, const void *code_addr, - unsigned int code_size, PyCodeObject* code); + size_t code_size, PyCodeObject* code); int (*free_state)(void* state); void *state; Py_ssize_t code_padding; @@ -88,6 +88,9 @@ struct _ceval_runtime_state { struct trampoline_api_st trampoline_api; FILE *map_file; Py_ssize_t persist_after_fork; + _PyFrameEvalFunction prev_eval_frame; + Py_ssize_t trampoline_refcount; + int code_watcher_id; #else int _not_used; #endif @@ -98,7 +101,6 @@ struct _ceval_runtime_state { // For example, we use a preallocated array // for the list of pending calls. struct _pending_calls pending_mainthread; - PyMutex sys_trace_profile_mutex; }; @@ -175,62 +177,77 @@ struct gc_generation { generations */ }; -struct gc_collection_stats { - /* number of collected objects */ - Py_ssize_t collected; - /* total number of uncollectable objects (put into gc.garbage) */ - Py_ssize_t uncollectable; -}; - /* Running stats per generation */ struct gc_generation_stats { + PyTime_t ts_start; + PyTime_t ts_stop; /* total number of collections */ Py_ssize_t collections; /* total number of collected objects */ Py_ssize_t collected; /* total number of uncollectable objects (put into gc.garbage) */ Py_ssize_t uncollectable; + // Total number of objects considered for collection and traversed: + Py_ssize_t candidates; + // Total duration of the collection in seconds: + double duration; + /* heap_size on the start of the collection */ + Py_ssize_t heap_size; }; -enum _GCPhase { - GC_PHASE_MARK = 0, - GC_PHASE_COLLECT = 1 +#ifdef Py_GIL_DISABLED +#define GC_YOUNG_STATS_SIZE 1 +#define GC_OLD_STATS_SIZE 1 +#else +#define GC_YOUNG_STATS_SIZE 11 +#define GC_OLD_STATS_SIZE 3 +#endif +struct gc_young_stats_buffer { + struct gc_generation_stats items[GC_YOUNG_STATS_SIZE]; + int8_t index; +}; + +struct gc_old_stats_buffer { + struct gc_generation_stats items[GC_OLD_STATS_SIZE]; + int8_t index; }; /* If we change this, we need to change the default value in the - signature of gc.collect. */ + signature of gc.collect and change the size of PyStats.gc_stats */ #define NUM_GENERATIONS 3 -struct _gc_runtime_state { - /* List of objects that still need to be cleaned up, singly linked - * via their gc headers' gc_prev pointers. */ - PyObject *trash_delete_later; - /* Current call-stack depth of tp_dealloc calls. */ - int trash_delete_nesting; +struct gc_stats { + struct gc_young_stats_buffer young; + struct gc_old_stats_buffer old[2]; +}; +struct _gc_runtime_state { /* Is automatic collection enabled? */ int enabled; int debug; /* linked lists of container objects */ +#ifndef Py_GIL_DISABLED + struct gc_generation generations[NUM_GENERATIONS]; +#else struct gc_generation young; struct gc_generation old[2]; +#endif /* a permanent generation which won't be collected */ struct gc_generation permanent_generation; - struct gc_generation_stats generation_stats[NUM_GENERATIONS]; + struct gc_stats *generation_stats; /* true if we are currently running the collector */ int collecting; + // The frame that started the current collection. It might be NULL even when + // collecting (if no Python frame is running): + _PyInterpreterFrame *frame; /* list of uncollectable objects */ PyObject *garbage; /* a list of callbacks to be invoked when collection is performed */ PyObject *callbacks; + /* The number of live objects. */ Py_ssize_t heap_size; - Py_ssize_t work_to_do; - /* Which of the old spaces is the visited space */ - int visited_space; - int phase; -#ifdef Py_GIL_DISABLED /* This is the number of objects that survived the last full collection. It approximates the number of long lived objects tracked by the GC. @@ -243,21 +260,31 @@ struct _gc_runtime_state { the first time. */ Py_ssize_t long_lived_pending; +#ifdef Py_GIL_DISABLED /* True if gc.freeze() has been used. */ int freeze_active; - - /* Memory usage of the process (RSS + swap) after last GC. */ - Py_ssize_t last_mem; - - /* This accumulates the new object count whenever collection is deferred - due to the RSS increase condition not being meet. Reset on collection. */ - Py_ssize_t deferred_count; - - /* Mutex held for gc_should_collect_mem_usage(). */ - PyMutex mutex; +#else + PyGC_Head *generation0; #endif }; +#ifndef Py_GIL_DISABLED +#define GC_GENERATION_INIT \ + .generations = { \ + { .threshold = 2000, }, \ + { .threshold = 10, }, \ + { .threshold = 10, }, \ + }, \ + .heap_size = 0, +#else +#define GC_GENERATION_INIT \ + .young = { .threshold = 2000, }, \ + .old = { \ + { .threshold = 10, }, \ + { .threshold = 10, }, \ + }, +#endif + #include "pycore_gil.h" // struct _gil_runtime_state /**** Import ********/ @@ -281,8 +308,6 @@ struct _import_runtime_state { Modules are added there and looked up in _imp.find_extension(). */ struct _Py_hashtable_t *hashtable; } extensions; - /* Package context -- the full module name for package imports */ - const char * pkgcontext; }; struct _import_state { @@ -318,6 +343,22 @@ struct _import_state { int dlopenflags; #endif PyObject *import_func; + PyObject *lazy_import_func; + int lazy_imports_mode; + PyObject *lazy_imports_filter; + PyObject *lazy_importing_modules; + // The set stored in sys.lazy_modules if values that have been + // lazily imported. This value is only for debugging/introspection + // purposes and is not used by the runtime. + PyObject *lazy_modules; + // A dict mapping package names to a set of submodule names that + // have been imported lazily from packages which have been imported + // lazily. When the package is reified we need to add a + // LazyImportObject which refers to the submodule on the module. + PyObject *lazy_pending_submodules; +#ifdef Py_GIL_DISABLED + PyMutex lazy_mutex; +#endif /* The global import lock. */ _PyRecursiveMutex lock; /* diagnostic info in PyImport_ImportModuleLevelObject() */ @@ -392,6 +433,28 @@ typedef struct _rare_events { uint8_t func_modification; } _rare_events; +// Optimization configuration for the interpreter. +// This groups all thresholds and optimization flags for both JIT and interpreter. +typedef struct _PyOptimizationConfig { + // Interpreter optimization thresholds + uint16_t jump_backward_initial_value; + uint16_t jump_backward_initial_backoff; + + uint16_t resume_initial_value; + uint16_t resume_initial_backoff; + + // JIT optimization thresholds + uint16_t side_exit_initial_value; + uint16_t side_exit_initial_backoff; + + // Trace fitness thresholds + uint16_t fitness_initial; + + // Optimization flags + bool specialization_enabled; + bool uops_optimize_enabled; +} _PyOptimizationConfig; + struct Bigint { struct Bigint *next; @@ -465,8 +528,13 @@ struct _py_func_state { /****** type state *********/ /* For now we hard-code this to a value for which we are confident - all the static builtin types will fit (for all builds). */ -#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES 200 + all the static builtin types will fit (for all builds). + If you add a new static type to the standard library, you may have to + update one of these numbers. + */ +#define _Py_NUM_MANAGED_PREINITIALIZED_TYPES 120 +#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES \ + (_Py_NUM_MANAGED_PREINITIALIZED_TYPES + 83) #define _Py_MAX_MANAGED_STATIC_EXT_TYPES 10 #define _Py_MAX_MANAGED_STATIC_TYPES \ (_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES + _Py_MAX_MANAGED_STATIC_EXT_TYPES) @@ -677,11 +745,6 @@ struct _Py_interp_cached_objects { /* object.__reduce__ */ PyObject *objreduce; -#ifndef Py_GIL_DISABLED - /* resolve_slotdups() */ - PyObject *type_slots_pname; - pytype_slotdef *type_slots_ptrs[MAX_EQUIV]; -#endif /* TypeVar and related types */ PyTypeObject *generic_type; @@ -691,6 +754,13 @@ struct _Py_interp_cached_objects { PyTypeObject *paramspecargs_type; PyTypeObject *paramspeckwargs_type; PyTypeObject *constevaluator_type; + + /* Descriptors for __dict__ and __weakref__ */ +#ifdef Py_GIL_DISABLED + PyMutex descriptor_mutex; +#endif + PyObject *dict_descriptor; + PyObject *weakref_descriptor; }; struct _Py_interp_static_objects { @@ -758,6 +828,9 @@ struct _Py_unique_id_pool { #endif +typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); + +#define _PyInterpreterGuard_GUARDS_NOT_ALLOWED UINTPTR_MAX /* PyInterpreterState holds the global state for one of the runtime's interpreters. Typically the initial (main) interpreter is the only one. @@ -771,12 +844,6 @@ struct _is { * and should be placed at the beginning. */ struct _ceval_state ceval; - /* This structure is carefully allocated so that it's correctly aligned - * to avoid undefined behaviors during LOAD and STORE. The '_malloced' - * field stores the allocated pointer address that will later be freed. - */ - void *_malloced; - PyInterpreterState *next; int64_t id; @@ -868,6 +935,7 @@ struct _is { PyObject *builtins_copy; // Initialized to _PyEval_EvalFrameDefault(). _PyFrameEvalFunction eval_frame; + int eval_frame_allow_specialization; PyFunction_WatchCallback func_watchers[FUNC_MAX_WATCHERS]; // One bit is set for each non-NULL entry in func_watchers @@ -939,18 +1007,26 @@ struct _is { struct ast_state ast; struct types_state types; struct callable_cache callable_cache; - PyObject *common_consts[NUM_COMMON_CONSTANTS]; + _PyStackRef common_consts[NUM_COMMON_CONSTANTS]; bool jit; - struct _PyExecutorObject *executor_list_head; + bool compiling; + + // Optimization configuration (thresholds and flags for JIT and interpreter) + _PyOptimizationConfig opt_config; + _PyBloomFilter *executor_blooms; // Contiguous bloom filter array + struct _PyExecutorObject **executor_ptrs; // Corresponding executor pointer array + size_t executor_count; // Number of valid executors + size_t executor_capacity; // Array capacity struct _PyExecutorObject *executor_deletion_list_head; - int executor_deletion_list_remaining_capacity; - size_t trace_run_counter; + struct _PyExecutorObject *cold_executor; + struct _PyExecutorObject *cold_dynamic_executor; + size_t executor_creation_counter; _rare_events rare_events; PyDict_WatchCallback builtins_dict_watcher; _Py_GlobalMonitors monitors; - bool sys_profile_initialized; - bool sys_trace_initialized; + _PyOnceFlag sys_profile_once_flag; + _PyOnceFlag sys_trace_once_flag; Py_ssize_t sys_profiling_threads; /* Count of threads with c_profilefunc set */ Py_ssize_t sys_tracing_threads; /* Count of threads with c_tracefunc set */ PyObject *monitoring_callables[PY_MONITORING_TOOL_IDS][_PY_MONITORING_EVENTS]; @@ -970,6 +1046,23 @@ struct _is { # endif #endif +#ifdef Py_STATS + // true if recording of pystats is on, this is used when new threads + // are created to decide if recording should be on for them + int pystats_enabled; + // allocated when (and if) stats are first enabled + PyStats *pystats_struct; +#ifdef Py_GIL_DISABLED + // held when pystats related interpreter state is being updated + PyMutex pystats_mutex; +#endif +#endif + + // The number of remaining finalization guards. + // If this is _PyInterpreterGuard_GUARDS_NOT_ALLOWED, then finalization + // guards can no longer be created. + uintptr_t finalization_guards; + /* the initial PyInterpreterState.threads.head */ _PyThreadStateImpl _initial_thread; // _initial_thread should be the last field of PyInterpreterState. diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index 2ee3696317c7270..3608a7f3bce894a 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -24,6 +24,36 @@ static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { return (PyCodeObject *)executable; } +// Similar to _PyFrame_GetCode(), but return NULL if the frame is invalid or +// freed. Used by dump_frame() in Python/traceback.c. The function uses +// heuristics to detect freed memory, it's not 100% reliable. +static inline PyCodeObject* _Py_NO_SANITIZE_THREAD +_PyFrame_SafeGetCode(_PyInterpreterFrame *f) +{ + // globals and builtins may be NULL on a legit frame, but it's unlikely. + // It's more likely that it's a sign of an invalid frame. + if (f->f_globals == NULL || f->f_builtins == NULL) { + return NULL; + } + + if (PyStackRef_IsNull(f->f_executable)) { + return NULL; + } + void *ptr; + memcpy(&ptr, &f->f_executable, sizeof(f->f_executable)); + if (_PyMem_IsPtrFreed(ptr)) { + return NULL; + } + PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable); + if (_PyObject_IsFreed(executable)) { + return NULL; + } + if (!PyCode_Check(executable)) { + return NULL; + } + return (PyCodeObject *)executable; +} + static inline _Py_CODEUNIT * _PyFrame_GetBytecode(_PyInterpreterFrame *f) { @@ -37,6 +67,31 @@ _PyFrame_GetBytecode(_PyInterpreterFrame *f) #endif } +// Similar to PyUnstable_InterpreterFrame_GetLasti(), but return NULL if the +// frame is invalid or freed. Used by dump_frame() in Python/traceback.c. The +// function uses heuristics to detect freed memory, it's not 100% reliable. +static inline int _Py_NO_SANITIZE_THREAD +_PyFrame_SafeGetLasti(struct _PyInterpreterFrame *f) +{ + // Code based on _PyFrame_GetBytecode() but replace _PyFrame_GetCode() + // with _PyFrame_SafeGetCode(). + PyCodeObject *co = _PyFrame_SafeGetCode(f); + if (co == NULL) { + return -1; + } + + _Py_CODEUNIT *bytecode; +#ifdef Py_GIL_DISABLED + _PyCodeArray *tlbc = _PyCode_GetTLBCArray(co); + assert(f->tlbc_index >= 0 && f->tlbc_index < tlbc->size); + bytecode = (_Py_CODEUNIT *)tlbc->entries[f->tlbc_index]; +#else + bytecode = _PyCode_CODE(co); +#endif + + return (int)(f->instr_ptr - bytecode) * sizeof(_Py_CODEUNIT); +} + static inline PyFunctionObject *_PyFrame_GetFunction(_PyInterpreterFrame *f) { PyObject *func = PyStackRef_AsPyObjectBorrow(f->f_funcobj); assert(PyFunction_Check(func)); @@ -47,10 +102,10 @@ static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) { return (f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); } -static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f) { +static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f, int depth) { assert(f->stackpointer > _PyFrame_Stackbase(f)); - assert(!PyStackRef_IsNull(f->stackpointer[-1])); - return f->stackpointer[-1]; + assert(!PyStackRef_IsNull(f->stackpointer[-depth])); + return f->stackpointer[-depth]; } static inline _PyStackRef _PyFrame_StackPop(_PyInterpreterFrame *f) { @@ -94,6 +149,12 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * int stacktop = (int)(src->stackpointer - src->localsplus); assert(stacktop >= 0); dest->stackpointer = dest->localsplus + stacktop; + // visited is GC bookkeeping for the current stack walk, not frame state. + dest->visited = 0; +#ifdef Py_DEBUG + dest->stackpointer_valid = src->stackpointer_valid; + dest->lltrace = src->lltrace; +#endif for (int i = 0; i < stacktop; i++) { dest->localsplus[i] = PyStackRef_MakeHeapSafe(src->localsplus[i]); } @@ -147,6 +208,7 @@ _PyFrame_Initialize( frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; #ifdef Py_DEBUG + frame->stackpointer_valid = 1; frame->lltrace = 0; #endif @@ -170,21 +232,51 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) static inline _PyStackRef* _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { - assert(frame->stackpointer != NULL); - _PyStackRef *sp = frame->stackpointer; -#ifndef NDEBUG - frame->stackpointer = NULL; -#endif - return sp; + return frame->stackpointer; } static inline void _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) { - assert(frame->stackpointer == NULL); frame->stackpointer = stack_pointer; } +static inline void +_PyFrame_StackPointerValidate(_PyInterpreterFrame *frame) +{ +#ifdef Py_DEBUG +/* Avoid bloating the JIT code */ +#ifndef _Py_JIT + assert(frame->stackpointer_valid == 0); +#endif + frame->stackpointer_valid = 1; +#endif +} + +static inline void +_PyFrame_StackPointerInvalidate(_PyInterpreterFrame *frame) +{ +#ifdef Py_DEBUG +/* Avoid bloating the JIT code */ +#ifndef _Py_JIT + assert(frame->stackpointer_valid == 1); +#endif + frame->stackpointer_valid = 0; +#endif +} + +static inline void +_PyFrame_StackAssertInvalid(_PyInterpreterFrame *frame) +{ +#ifdef Py_DEBUG +/* Avoid bloating the JIT code */ +#ifndef _Py_JIT + assert(frame->stackpointer_valid == 0); +#endif +#endif +} + + /* Determine whether a frame is incomplete. * A frame is incomplete if it is part way through * creating cell objects or a generator or coroutine. @@ -222,9 +314,23 @@ _PyThreadState_GetFrame(PyThreadState *tstate) return _PyFrame_GetFirstComplete(tstate->current_frame); } +// Update last_profiled_frame for remote profiler frame caching. +// Only update if we're removing the exact frame that was last profiled. +// This avoids corrupting the cache when transient frames (called and returned +// between profiler samples) update last_profiled_frame to addresses the +// profiler never saw. +#define _PyThreadState_UpdateLastProfiledFrame(tstate, frame, previous) \ + do { \ + PyThreadState *tstate_ = (tstate); \ + _PyInterpreterFrame *frame_ = (frame); \ + if (tstate_->last_profiled_frame == frame_) { \ + tstate_->last_profiled_frame = (previous); \ + } \ + } while (0) + /* For use by _PyFrame_GetFrameObject Do not call directly. */ -PyFrameObject * +PyAPI_FUNC(PyFrameObject *) _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame); /* Gets the PyFrameObject for this frame, lazily @@ -242,7 +348,8 @@ _PyFrame_GetFrameObject(_PyInterpreterFrame *frame) return _PyFrame_MakeAndSetFrameObject(frame); } -void +// Exported for external JIT support +PyAPI_FUNC(void) _PyFrame_ClearLocals(_PyInterpreterFrame *frame); /* Clears all references in the frame. @@ -253,8 +360,10 @@ _PyFrame_ClearLocals(_PyInterpreterFrame *frame); * in the frame. * take should be set to 1 for heap allocated * frames like the ones in generators and coroutines. + * + * Exported for external JIT support */ -void + PyAPI_FUNC(void) _PyFrame_ClearExceptCode(_PyInterpreterFrame * frame); int @@ -278,7 +387,8 @@ _PyThreadState_HasStackSpace(PyThreadState *tstate, int size) size < tstate->datastack_limit - tstate->datastack_top; } -extern _PyInterpreterFrame * +// Exported for external JIT support +PyAPI_FUNC(_PyInterpreterFrame *) _PyThreadState_PushFrame(PyThreadState *tstate, size_t size); PyAPI_FUNC(void) _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame *frame); @@ -328,6 +438,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; #ifdef Py_DEBUG + frame->stackpointer_valid = 1; frame->lltrace = 0; #endif frame->return_offset = 0; @@ -340,6 +451,10 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func, size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous); +PyAPI_FUNC(_PyInterpreterFrame *) +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 835b8e58194863f..4d267e35504b94d 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -24,7 +24,6 @@ enum _frameowner { FRAME_OWNED_BY_GENERATOR = 1, FRAME_OWNED_BY_FRAME_OBJECT = 2, FRAME_OWNED_BY_INTERPRETER = 3, - FRAME_OWNED_BY_CSTACK = 4, }; struct _PyInterpreterFrame { @@ -45,7 +44,8 @@ struct _PyInterpreterFrame { char owner; #ifdef Py_DEBUG uint8_t visited:1; - uint8_t lltrace:7; + uint8_t stackpointer_valid:1; + uint8_t lltrace:6; #else uint8_t visited; #endif diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 39c2a30f6e979de..59a7b16073f886c 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -18,8 +18,9 @@ #define INTRINSIC_TYPEVARTUPLE 9 #define INTRINSIC_SUBSCRIPT_GENERIC 10 #define INTRINSIC_TYPEALIAS 11 +#define INTRINSIC_BUILD_FROZENSET 12 -#define MAX_INTRINSIC_1 11 +#define MAX_INTRINSIC_1 12 /* Binary Functions: */ diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h index 8a88cbf607ba4bc..9ce6ec97b6090e4 100644 --- a/Include/internal/pycore_jit.h +++ b/Include/internal/pycore_jit.h @@ -13,12 +13,26 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +/* To be able to reason about code layout and branches, keep code size below 1 MB */ +#define PY_MAX_JIT_CODE_SIZE ((1 << 20)-1) + #ifdef _Py_JIT -typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); +typedef _Py_CODEUNIT *(*jit_func)( + _PyExecutorObject *executor, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, + _PyStackRef _tos_cache0, _PyStackRef _tos_cache1, _PyStackRef _tos_cache2 +); + +_Py_CODEUNIT *_PyJIT_Entry( + _PyExecutorObject *executor, _PyInterpreterFrame *frame, + _PyStackRef *stack_pointer, PyThreadState *tstate +); int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length); void _PyJIT_Free(_PyExecutorObject *executor); +PyAPI_FUNC(int) _PyJIT_AddressInJitCode(PyInterpreterState *interp, uintptr_t addr); +PyAPI_FUNC(void) _Py_jit_assert_within_stack_bounds(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int lineno); +PyAPI_FUNC(int) _Py_jit_assertion_failure(int line); #endif // _Py_JIT diff --git a/Include/internal/pycore_jit_publish.h b/Include/internal/pycore_jit_publish.h new file mode 100644 index 000000000000000..7ee77f5c996fafe --- /dev/null +++ b/Include/internal/pycore_jit_publish.h @@ -0,0 +1,31 @@ +#ifndef Py_INTERNAL_JIT_PUBLISH_H +#define Py_INTERNAL_JIT_PUBLISH_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include + +typedef struct _PyJitCodeRegistration _PyJitCodeRegistration; + +#ifdef _Py_JIT + +/* Publish JIT code to optional tooling backends. + * + * The return value is a backend-specific deregistration handle, not a + * success/failure indicator. NULL means there is nothing to unregister later: + * perf does not need a handle, and GDB/GNU backtrace registration failures + * are intentionally non-fatal because tooling support must not make JIT + * compilation fail. + */ +_PyJitCodeRegistration *_PyJit_RegisterCode(const void *code_addr, + size_t code_size, + const char *entry, + const char *filename); + +void _PyJit_UnregisterCode(_PyJitCodeRegistration *registration); + +#endif // _Py_JIT + +#endif // Py_INTERNAL_JIT_PUBLISH_H diff --git a/Include/internal/pycore_jit_unwind.h b/Include/internal/pycore_jit_unwind.h new file mode 100644 index 000000000000000..7099b88812ce7be --- /dev/null +++ b/Include/internal/pycore_jit_unwind.h @@ -0,0 +1,81 @@ +#ifndef Py_INTERNAL_JIT_UNWIND_H +#define Py_INTERNAL_JIT_UNWIND_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include +#include + +#if defined(_Py_JIT) && defined(__linux__) && defined(__ELF__) +# define PY_HAVE_JIT_GDB_UNWIND +# if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \ + defined(_Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION) +# define PY_HAVE_JIT_GNU_BACKTRACE_UNWIND +# endif +#endif + +#if defined(PY_HAVE_PERF_TRAMPOLINE) \ + || defined(PY_HAVE_JIT_GDB_UNWIND) \ + || defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) + +#if defined(PY_HAVE_JIT_GDB_UNWIND) +extern PyMutex _Py_jit_debug_mutex; +#endif + +/* DWARF exception-handling pointer encodings shared by JIT unwind users. */ +enum { + DWRF_EH_PE_absptr = 0x00, + DWRF_EH_PE_omit = 0xff, + + /* Data type encodings */ + DWRF_EH_PE_uleb128 = 0x01, + DWRF_EH_PE_udata2 = 0x02, + DWRF_EH_PE_udata4 = 0x03, + DWRF_EH_PE_udata8 = 0x04, + DWRF_EH_PE_sleb128 = 0x09, + DWRF_EH_PE_sdata2 = 0x0a, + DWRF_EH_PE_sdata4 = 0x0b, + DWRF_EH_PE_sdata8 = 0x0c, + DWRF_EH_PE_signed = 0x08, + + /* Reference type encodings */ + DWRF_EH_PE_pcrel = 0x10, + DWRF_EH_PE_textrel = 0x20, + DWRF_EH_PE_datarel = 0x30, + DWRF_EH_PE_funcrel = 0x40, + DWRF_EH_PE_aligned = 0x50, + DWRF_EH_PE_indirect = 0x80 +}; + +/* Return the size of the generated .eh_frame data for the given encoding. */ +size_t _PyJitUnwind_EhFrameSize(int absolute_addr); + +/* + * Build DWARF .eh_frame data for JIT code; returns size written or 0 on error. + * absolute_addr selects the FDE address encoding: + * - 0: PC-relative offsets (perf jitdump synthesized DSO). + * - nonzero: absolute addresses (GDB JIT in-memory ELF). + */ +size_t _PyJitUnwind_BuildEhFrame(uint8_t *buffer, size_t buffer_size, + const void *code_addr, size_t code_size, + int absolute_addr); + +void *_PyJitUnwind_GdbRegisterCode(const void *code_addr, + size_t code_size, + const char *entry, + const char *filename); + +void _PyJitUnwind_GdbUnregisterCode(void *handle); + +#if defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) +void *_PyJitUnwind_GnuBacktraceRegisterCode(const void *code_addr, + size_t code_size); + +void _PyJitUnwind_GnuBacktraceUnregisterCode(void *handle); +#endif + +#endif // JIT unwind support + +#endif // Py_INTERNAL_JIT_UNWIND_H diff --git a/Include/internal/pycore_lazyimportobject.h b/Include/internal/pycore_lazyimportobject.h new file mode 100644 index 000000000000000..b81e4211b08ff39 --- /dev/null +++ b/Include/internal/pycore_lazyimportobject.h @@ -0,0 +1,35 @@ +// Lazy object interface. + +#ifndef Py_INTERNAL_LAZYIMPORTOBJECT_H +#define Py_INTERNAL_LAZYIMPORTOBJECT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +PyAPI_DATA(PyTypeObject) PyLazyImport_Type; +#define PyLazyImport_CheckExact(op) Py_IS_TYPE((op), &PyLazyImport_Type) + +typedef struct { + PyObject_HEAD + PyObject *lz_builtins; + PyObject *lz_from; + PyObject *lz_attr; + // Frame information for the original import location. + PyCodeObject *lz_code; // Code object where the lazy import was created. + int lz_instr_offset; // Instruction offset where the lazy import was created. +} PyLazyImportObject; + + +PyAPI_FUNC(PyObject *) _PyLazyImport_GetName(PyObject *lazy_import); +PyAPI_FUNC(PyObject *) _PyLazyImport_New( + struct _PyInterpreterFrame *frame, PyObject *import_func, PyObject *from, PyObject *attr); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_LAZYIMPORTOBJECT_H diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index ffbcebdb7dfb50d..df0d00f752573b5 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -14,15 +14,17 @@ extern "C" { PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *); PyAPI_FUNC(PyObject) *_PyList_SliceSubscript(PyObject*, PyObject*); +PyAPI_FUNC(PyObject *) _PyList_BinarySlice(PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyList_Concat(PyObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); // _PyList_GetItemRef should be used only when the object is known as a list // because it doesn't raise TypeError when the object is not a list, whereas PyList_GetItemRef does. -extern PyObject* _PyList_GetItemRef(PyListObject *, Py_ssize_t i); +PyAPI_FUNC(PyObject *) _PyList_GetItemRef(PyListObject *, Py_ssize_t i); #ifdef Py_GIL_DISABLED // Returns -1 in case of races with other threads. -extern int _PyList_GetItemRefNoLock(PyListObject *, Py_ssize_t, _PyStackRef *); +PyAPI_FUNC(int) _PyList_GetItemRefNoLock(PyListObject *, Py_ssize_t, _PyStackRef *); #endif #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item) diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index c4e007e744ce0f7..e31d8b4e5c68c92 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -70,6 +70,9 @@ PyMutex_LockFlags(PyMutex *m, _PyLockFlags flags) // error messages) otherwise returns 0. extern int _PyMutex_TryUnlock(PyMutex *m); +// Yield the processor to other threads (e.g., sched_yield). +extern void _Py_yield(void); + // PyEvent is a one-time event notification typedef struct { diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 3c213783cd432b9..fb5622c99f7a135 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -64,7 +64,8 @@ PyAPI_FUNC(void) _PyLong_ExactDealloc(PyObject *self); # error "_PY_NSMALLPOSINTS must be greater than or equal to 257" #endif -#define _PY_IS_SMALL_INT(val) ((val) >= 0 && (val) < 256 && (val) < _PY_NSMALLPOSINTS) +#define _PY_IS_SMALL_INT(val) \ + (-_PY_NSMALLNEGINTS <= (val) && (val) < _PY_NSMALLPOSINTS) // Return a reference to the immortal zero singleton. // The function cannot return NULL. @@ -135,7 +136,7 @@ extern int _PyLong_FormatWriter( int alternate); extern char* _PyLong_FormatBytesWriter( - _PyBytesWriter *writer, + PyBytesWriter *writer, char *str, PyObject *obj, int base, @@ -231,6 +232,21 @@ _PyLong_IsPositive(const PyLongObject *op) return (op->long_value.lv_tag & SIGN_MASK) == 0; } +/* Return true if the argument is a small int */ +static inline bool +_PyLong_IsSmallInt(const PyLongObject *op) +{ + assert(PyLong_Check(op)); + bool is_small_int = (op->long_value.lv_tag & IMMORTALITY_BIT_MASK) != 0; + if (is_small_int) { + assert(PyLong_CheckExact(op)); + assert(_Py_IsImmortal(op)); + assert((_PyLong_IsCompact(op) + && _PY_IS_SMALL_INT(_PyLong_CompactValue(op)))); + } + return is_small_int; +} + static inline Py_ssize_t _PyLong_DigitCount(const PyLongObject *op) { @@ -270,6 +286,14 @@ _PyLong_SameSign(const PyLongObject *a, const PyLongObject *b) return (a->long_value.lv_tag & SIGN_MASK) == (b->long_value.lv_tag & SIGN_MASK); } +/* Initialize the tag of a freshly-allocated int. */ +static inline void +_PyLong_InitTag(PyLongObject *op) +{ + assert(PyLong_Check(op)); + op->long_value.lv_tag = SIGN_ZERO; /* non-immortal zero */ +} + #define TAG_FROM_SIGN_AND_SIZE(sign, size) \ ((uintptr_t)(1 - (sign)) | ((uintptr_t)(size) << NON_SIZE_BITS)) @@ -279,6 +303,7 @@ _PyLong_SetSignAndDigitCount(PyLongObject *op, int sign, Py_ssize_t size) assert(size >= 0); assert(-1 <= sign && sign <= 1); assert(sign != 0 || size == 0); + assert(!_PyLong_IsSmallInt(op)); op->long_value.lv_tag = TAG_FROM_SIGN_AND_SIZE(sign, size); } @@ -286,13 +311,16 @@ static inline void _PyLong_SetDigitCount(PyLongObject *op, Py_ssize_t size) { assert(size >= 0); + assert(!_PyLong_IsSmallInt(op)); op->long_value.lv_tag = (((size_t)size) << NON_SIZE_BITS) | (op->long_value.lv_tag & SIGN_MASK); } #define NON_SIZE_MASK ~(uintptr_t)((1 << NON_SIZE_BITS) - 1) static inline void -_PyLong_FlipSign(PyLongObject *op) { +_PyLong_FlipSign(PyLongObject *op) +{ + assert(!_PyLong_IsSmallInt(op)); unsigned int flipped_sign = 2 - (op->long_value.lv_tag & SIGN_MASK); op->long_value.lv_tag &= NON_SIZE_MASK; op->long_value.lv_tag |= flipped_sign; diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 347d9762f26bff9..6a8fa124ba38a7c 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -277,13 +277,31 @@ Known values: Python 3.14a7 3622 (Store annotations in different class dict keys) Python 3.14a7 3623 (Add BUILD_INTERPOLATION & BUILD_TEMPLATE opcodes) Python 3.14b1 3624 (Don't optimize LOAD_FAST when local is killed by DELETE_FAST) + Python 3.14b3 3625 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) + Python 3.14rc2 3626 (Fix missing exception handlers in logical expression) + Python 3.14rc3 3627 (Fix miscompilation of some module-level annotations) Python 3.15a0 3650 (Initial version) Python 3.15a1 3651 (Simplify LOAD_CONST) Python 3.15a1 3652 (Virtual iterators) Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) + Python 3.15a1 3654 (Fix missing exception handlers in logical expression) + Python 3.15a1 3655 (Fix miscompilation of some module-level annotations) + Python 3.15a1 3656 (Add TRACE_RECORD instruction, for platforms with switch based interpreter) + Python 3.15a4 3657 (Add BINARY_OP_SUBSCR_USTR_INT) + Python 3.15a4 3658 (Optimize bytecode for list/set called on genexp) + Python 3.15a4 3659 (Add CALL_FUNCTION_EX specialization) + Python 3.15a4 3660 (Change generator preamble code) + Python 3.15a4 3661 (Lazy imports IMPORT_NAME opcode changes) + Python 3.15a8 3662 (Add counter to RESUME) + Python 3.15a8 3663 (Merge GET_ITER and GET_YIELD_FROM_ITER. Modify SEND to make it a bit more like FOR_ITER) + Python 3.15a8 3664 (Fix __qualname__ for __annotate__ functions) + Python 3.15a8 3665 (Add FOR_ITER_VIRTUAL and GET_ITER specializations) + Python 3.15b1 3666 (Add SEND_VIRTUAL and SEND_ASYNC_GEN specializations) + Python 3.16a0 3700 (Initial version) + Python 3.16a0 3701 (Add CONSTANT_EMPTY_TUPLE to LOAD_COMMON_CONSTANT) - Python 3.16 will start with 3700 + Python 3.17 will start with 3750 Please don't copy-paste the same pre-release tag for new entries above!!! You should always use the *upcoming* tag. For example, if 3.12a6 came out @@ -294,7 +312,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3653 +#define PYC_MAGIC_NUMBER 3701 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_mmap.h b/Include/internal/pycore_mmap.h new file mode 100644 index 000000000000000..c117cbd16283da9 --- /dev/null +++ b/Include/internal/pycore_mmap.h @@ -0,0 +1,47 @@ +#ifndef Py_INTERNAL_MMAP_H +#define Py_INTERNAL_MMAP_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_pystate.h" + +#if defined(_Py_HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) +# include +# include +#endif + +#if defined(_Py_HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) +static inline int +_PyAnnotateMemoryMap(void *addr, size_t size, const char *name) +{ +#ifndef Py_DEBUG + if (!_Py_GetConfig()->dev_mode) { + return 0; + } +#endif + // The name length cannot exceed 80 (including the '\0'). + assert(strlen(name) < 80); + int res = prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, (unsigned long)addr, size, name); + if (res < 0) { + return -1; + } + return 0; +} +#else +static inline int +_PyAnnotateMemoryMap(void *Py_UNUSED(addr), size_t Py_UNUSED(size), const char *Py_UNUSED(name)) +{ + return 0; +} +#endif + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_MMAP_H diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index b170d7bce702c62..5bcfd17cec46271 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -1,5 +1,8 @@ #ifndef Py_INTERNAL_MODULEOBJECT_H #define Py_INTERNAL_MODULEOBJECT_H + +#include + #ifdef __cplusplus extern "C" { #endif @@ -16,32 +19,53 @@ extern int _PyModule_IsPossiblyShadowing(PyObject *); extern int _PyModule_IsExtension(PyObject *obj); +extern int _PyModule_InitModuleDictWatcher(PyInterpreterState *interp); + +typedef int (*_Py_modexecfunc)(PyObject *); + typedef struct { PyObject_HEAD PyObject *md_dict; - PyModuleDef *md_def; void *md_state; PyObject *md_weaklist; // for logging purposes after md_dict is cleared PyObject *md_name; + bool md_token_is_def; /* if true, `md_token` is the PyModuleDef */ #ifdef Py_GIL_DISABLED - void *md_gil; + bool md_requires_gil; #endif + Py_ssize_t md_state_size; + traverseproc md_state_traverse; + inquiry md_state_clear; + freefunc md_state_free; + void *md_token; + _Py_modexecfunc md_exec; /* only set if md_token_is_def is true */ } PyModuleObject; -static inline PyModuleDef* _PyModule_GetDef(PyObject *mod) { - assert(PyModule_Check(mod)); - return ((PyModuleObject *)mod)->md_def; +#define _PyModule_CAST(op) \ + (assert(PyModule_Check(op)), _Py_CAST(PyModuleObject*, (op))) + +static inline PyModuleDef *_PyModule_GetDefOrNull(PyObject *arg) { + PyModuleObject *mod = _PyModule_CAST(arg); + if (mod->md_token_is_def) { + return (PyModuleDef *)mod->md_token; + } + return NULL; +} + +// Get md_token. Used in _DuringGC functions; must have no side effects. +static inline PyModuleDef *_PyModule_GetToken(PyObject *arg) { + PyModuleObject *mod = _PyModule_CAST(arg); + return (PyModuleDef *)mod->md_token; } +// Get md_state. Used in _DuringGC functions; must have no side effects. static inline void* _PyModule_GetState(PyObject* mod) { - assert(PyModule_Check(mod)); - return ((PyModuleObject *)mod)->md_state; + return _PyModule_CAST(mod)->md_state; } static inline PyObject* _PyModule_GetDict(PyObject *mod) { - assert(PyModule_Check(mod)); - PyObject *dict = ((PyModuleObject *)mod) -> md_dict; + PyObject *dict = _PyModule_CAST(mod)->md_dict; // _PyModule_GetDict(mod) must not be used after calling module_clear(mod) assert(dict != NULL); return dict; // borrowed reference diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 40f8ca68c00b72c..98880cd83780dca 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -144,7 +144,7 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) new_refcnt = _Py_IMMORTAL_INITIAL_REFCNT; } # if SIZEOF_VOID_P > 4 - op->ob_refcnt = (PY_UINT32_T)new_refcnt; + op->ob_refcnt = (uint32_t)new_refcnt; # else op->ob_refcnt = new_refcnt; # endif @@ -252,25 +252,6 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) } } -static inline void -_Py_DECREF_NO_DEALLOC(PyObject *op) -{ - if (_Py_IsImmortal(op)) { - _Py_DECREF_IMMORTAL_STAT_INC(); - return; - } - _Py_DECREF_STAT_INC(); -#ifdef Py_REF_DEBUG - _Py_DEC_REFTOTAL(PyInterpreterState_Get()); -#endif - op->ob_refcnt--; -#ifdef Py_DEBUG - if (op->ob_refcnt <= 0) { - _Py_FatalRefcountError("Expected a positive remaining refcount"); - } -#endif -} - #else // TODO: implement Py_DECREF specializations for Py_GIL_DISABLED build static inline void @@ -279,24 +260,12 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) Py_DECREF(op); } -static inline void -_Py_DECREF_NO_DEALLOC(PyObject *op) -{ - Py_DECREF(op); -} - static inline int _Py_REF_IS_MERGED(Py_ssize_t ob_ref_shared) { return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_MERGED; } -static inline int -_Py_REF_IS_QUEUED(Py_ssize_t ob_ref_shared) -{ - return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_QUEUED; -} - // Merge the local and shared reference count fields and add `extra` to the // refcount when merging. Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra); @@ -496,6 +465,9 @@ static inline void Py_DECREF_MORTAL_SPECIALIZED(PyObject *op, destructor destruc #define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) Py_DECREF_MORTAL_SPECIALIZED(_PyObject_CAST(op), destruct) #endif +#else // Py_GIL_DISABLED +# define Py_DECREF_MORTAL(op) Py_DECREF(op) +# define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) Py_DECREF(op) #endif /* Inline functions trading binary compatibility for speed: @@ -858,13 +830,17 @@ _PyObject_IS_GC(PyObject *obj) && (type->tp_is_gc == NULL || type->tp_is_gc(obj))); } -// Fast inlined version of PyObject_Hash() -static inline Py_hash_t -_PyObject_HashFast(PyObject *op) +// Fast inlined version of PyObject_Hash(). Dictionaries are very +// likely to include string keys (class and instance attributes, +// json, ...) so we include a fast path for strings. +// This function should not be used in a collection if str is not +// very likely, since it is slower than PyObject_Hash() on types +// other than str. See gh-137759. +static inline Py_ALWAYS_INLINE Py_hash_t +_PyObject_HashDictKey(PyObject *op) { if (PyUnicode_CheckExact(op)) { - Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED( - _PyASCIIObject_CAST(op)->hash); + Py_hash_t hash = PyUnstable_Unicode_GET_CACHED_HASH(op); if (hash != -1) { return hash; } @@ -902,14 +878,16 @@ PyAPI_FUNC(PyObject *) _PyType_NewManagedObject(PyTypeObject *type); extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int); -extern int _PyObject_SetAttributeErrorContext(PyObject *v, PyObject* name); +// Exported for external JIT support +PyAPI_FUNC(int) _PyObject_SetAttributeErrorContext(PyObject *v, PyObject* name); void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyObject *name, PyObject *value); extern bool _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr); -extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, unsigned int *); // Internal API to look for a name through the MRO. @@ -918,9 +896,13 @@ extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, extern unsigned int _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef *out); -PyAPI_FUNC(int) _PyObject_GetMethodStackRef(PyThreadState *ts, PyObject *obj, +extern int _PyObject_GetMethodStackRef(PyThreadState *ts, _PyStackRef *self, PyObject *name, _PyStackRef *method); +// Like PyObject_GetAttr but returns a _PyStackRef. For types, this can +// return a deferred reference to reduce reference count contention. +PyAPI_FUNC(_PyStackRef) _PyObject_GetAttrStackRef(PyObject *obj, PyObject *name); + // Cache the provided init method in the specialization cache of type if the // provided type version matches the current version of the type. // @@ -929,7 +911,9 @@ PyAPI_FUNC(int) _PyObject_GetMethodStackRef(PyThreadState *ts, PyObject *obj, // deferred reference counting. // // Returns 1 if the value was cached or 0 otherwise. -extern int _PyType_CacheInitForSpecialization(PyHeapTypeObject *type, +// +// Exported for external JIT support +PyAPI_FUNC(int) _PyType_CacheInitForSpecialization(PyHeapTypeObject *type, PyObject *init, unsigned int tp_version); @@ -1031,7 +1015,8 @@ enum _PyAnnotateFormat { _Py_ANNOTATE_FORMAT_STRING = 4, }; -int _PyObject_SetDict(PyObject *obj, PyObject *value); +extern int _PyObject_SetDict(PyObject *obj, PyObject *value); +extern int _PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict); #ifndef Py_GIL_DISABLED static inline Py_ALWAYS_INLINE void _Py_INCREF_MORTAL(PyObject *op) @@ -1045,8 +1030,14 @@ static inline Py_ALWAYS_INLINE void _Py_INCREF_MORTAL(PyObject *op) } #endif } +#else +# define _Py_INCREF_MORTAL(op) Py_INCREF(op) #endif +/* Utility for the tp_traverse slot of mutable heap types that have no other + * references. */ +PyAPI_FUNC(int) _PyObject_VisitType(PyObject *op, visitproc visit, void *arg); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index a7ba8f340737aab..f6b4bd90a234554 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -14,6 +14,12 @@ typedef unsigned int pymem_uint; /* assuming >= 16 bits */ #undef uint #define uint pymem_uint +/* NOTE: the following overviews were in the initial checkin, in 1998. In + * 2026, they're still helpful, but some details have changed. For example, + * we now use 32 size classes 16 bytes apart, and an arena is generally at + * least 1MB. Use sys._debugmallocstats() to see exact current details for + * the specific version of CPython used. + */ /* An object allocator for Python. @@ -208,7 +214,11 @@ typedef unsigned int pymem_uint; /* assuming >= 16 bits */ * mappings to reduce heap fragmentation. */ #ifdef USE_LARGE_ARENAS -#define ARENA_BITS 20 /* 1 MiB */ +# ifdef PYMALLOC_USE_HUGEPAGES +# define ARENA_BITS 21 /* 2 MiB */ +# else +# define ARENA_BITS 20 /* 1 MiB */ +# endif #else #define ARENA_BITS 18 /* 256 KiB */ #endif @@ -469,7 +479,7 @@ nfp free pools in usable_arenas. */ /* How many arena_objects do we initially allocate? - * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4MB before growing the + * 16 = can allocate 16 arenas = 16 * ARENA_SIZE before growing the * `arenas` vector. */ #define INITIAL_ARENA_OBJECTS 16 @@ -512,7 +522,11 @@ struct _obmalloc_mgmt { memory address bit allocation for keys - 64-bit pointers, IGNORE_BITS=0 and 2^20 arena size: + ARENA_BITS is configurable: 20 (1 MiB) by default on 64-bit, or + 21 (2 MiB) when PYMALLOC_USE_HUGEPAGES is enabled. All bit widths + below are derived from ARENA_BITS automatically. + + 64-bit pointers, IGNORE_BITS=0 and 2^20 arena size (default): 15 -> MAP_TOP_BITS 15 -> MAP_MID_BITS 14 -> MAP_BOT_BITS @@ -520,6 +534,14 @@ struct _obmalloc_mgmt { ---- 64 + 64-bit pointers, IGNORE_BITS=0 and 2^21 arena size (hugepages): + 15 -> MAP_TOP_BITS + 15 -> MAP_MID_BITS + 13 -> MAP_BOT_BITS + 21 -> ideal aligned arena + ---- + 64 + 64-bit pointers, IGNORE_BITS=16, and 2^20 arena size: 16 -> IGNORE_BITS 10 -> MAP_TOP_BITS @@ -675,7 +697,11 @@ struct _obmalloc_state { /* Allocate memory directly from the O/S virtual memory system, - * where supported. Otherwise fallback on malloc */ + * where supported. Otherwise fallback on malloc. + * + * Large-page and huge-page backends may round the mapped size up + * internally, so pass the original requested size back to + * _PyObject_VirtualFree(). */ void *_PyObject_VirtualAlloc(size_t size); void _PyObject_VirtualFree(void *, size_t size); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index dd1bf2d1d2b51a9..f145b7f034c52c0 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -66,6 +66,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2; case BINARY_OP_SUBSCR_TUPLE_INT: return 2; + case BINARY_OP_SUBSCR_USTR_INT: + return 2; case BINARY_OP_SUBTRACT_FLOAT: return 2; case BINARY_OP_SUBTRACT_INT: @@ -106,6 +108,10 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + oparg; case CALL_BUILTIN_O: return 2 + oparg; + case CALL_EX_NON_PY_GENERAL: + return 4; + case CALL_EX_PY: + return 4; case CALL_FUNCTION_EX: return 4; case CALL_INTRINSIC_1: @@ -151,7 +157,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case CHECK_EXC_MATCH: return 2; case CLEANUP_THROW: - return 3; + return 4; case COMPARE_OP: return 2; case COMPARE_OP_FLOAT: @@ -193,7 +199,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case END_FOR: return 1; case END_SEND: - return 2; + return 3; case ENTER_EXECUTOR: return 0; case EXIT_INIT_CHECK: @@ -214,6 +220,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2; case FOR_ITER_TUPLE: return 2; + case FOR_ITER_VIRTUAL: + return 2; case GET_AITER: return 1; case GET_ANEXT: @@ -222,9 +230,11 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case GET_ITER: return 1; - case GET_LEN: + case GET_ITER_SELF: return 1; - case GET_YIELD_FROM_ITER: + case GET_ITER_VIRTUAL: + return 1; + case GET_LEN: return 1; case IMPORT_FROM: return 1; @@ -241,7 +251,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case INSTRUMENTED_END_FOR: return 3; case INSTRUMENTED_END_SEND: - return 2; + return 3; case INSTRUMENTED_FOR_ITER: return 2; case INSTRUMENTED_INSTRUCTION: @@ -420,14 +430,20 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case RESUME_CHECK: return 0; + case RESUME_CHECK_JIT: + return 0; case RETURN_GENERATOR: return 0; case RETURN_VALUE: return 1; case SEND: - return 2; + return 3; + case SEND_ASYNC_GEN: + return 3; case SEND_GEN: - return 2; + return 3; + case SEND_VIRTUAL: + return 3; case SETUP_ANNOTATIONS: return 0; case SETUP_CLEANUP: @@ -488,6 +504,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case TO_BOOL_STR: return 1; + case TRACE_RECORD: + return 0; case UNARY_INVERT: return 1; case UNARY_NEGATIVE: @@ -532,7 +550,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case BINARY_OP_EXTEND: return 1; case BINARY_OP_INPLACE_ADD_UNICODE: - return 0; + return 1; case BINARY_OP_MULTIPLY_FLOAT: return 1; case BINARY_OP_MULTIPLY_INT: @@ -549,6 +567,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case BINARY_OP_SUBSCR_TUPLE_INT: return 1; + case BINARY_OP_SUBSCR_USTR_INT: + return 1; case BINARY_OP_SUBTRACT_FLOAT: return 1; case BINARY_OP_SUBTRACT_INT: @@ -589,6 +609,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case CALL_BUILTIN_O: return 1; + case CALL_EX_NON_PY_GENERAL: + return 1; + case CALL_EX_PY: + return 0; case CALL_FUNCTION_EX: return 1; case CALL_INTRINSIC_1: @@ -608,7 +632,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CALL_LEN: return 1; case CALL_LIST_APPEND: - return 0; + return 1; case CALL_METHOD_DESCRIPTOR_FAST: return 1; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: @@ -634,7 +658,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CHECK_EXC_MATCH: return 2; case CLEANUP_THROW: - return 2; + return 3; case COMPARE_OP: return 1; case COMPARE_OP_FLOAT: @@ -697,6 +721,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 3; case FOR_ITER_TUPLE: return 3; + case FOR_ITER_VIRTUAL: + return 3; case GET_AITER: return 1; case GET_ANEXT: @@ -705,10 +731,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case GET_ITER: return 2; + case GET_ITER_SELF: + return 2; + case GET_ITER_VIRTUAL: + return 2; case GET_LEN: return 2; - case GET_YIELD_FROM_ITER: - return 1; case IMPORT_FROM: return 2; case IMPORT_NAME: @@ -788,7 +816,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case LOAD_ATTR_CLASS_WITH_METACLASS_CHECK: return 1 + (oparg & 1); case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: - return 1; + return 0; case LOAD_ATTR_INSTANCE_VALUE: return 1 + (oparg & 1); case LOAD_ATTR_METHOD_LAZY_DICT: @@ -903,14 +931,20 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case RESUME_CHECK: return 0; + case RESUME_CHECK_JIT: + return 0; case RETURN_GENERATOR: return 1; case RETURN_VALUE: return 1; case SEND: - return 2; + return 3; + case SEND_ASYNC_GEN: + return 3; case SEND_GEN: - return 1; + return 2; + case SEND_VIRTUAL: + return 3; case SETUP_ANNOTATIONS: return 0; case SETUP_CLEANUP: @@ -971,6 +1005,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case TO_BOOL_STR: return 1; + case TRACE_RECORD: + return 0; case UNARY_INVERT: return 1; case UNARY_NEGATIVE: @@ -1028,10 +1064,13 @@ enum InstructionFormat { #define HAS_ESCAPES_FLAG (512) #define HAS_EXIT_FLAG (1024) #define HAS_PURE_FLAG (2048) -#define HAS_PASSTHROUGH_FLAG (4096) -#define HAS_OPARG_AND_1_FLAG (8192) -#define HAS_ERROR_NO_POP_FLAG (16384) -#define HAS_NO_SAVE_IP_FLAG (32768) +#define HAS_SYNC_SP_FLAG (4096) +#define HAS_ERROR_NO_POP_FLAG (8192) +#define HAS_NO_SAVE_IP_FLAG (16384) +#define HAS_PERIODIC_FLAG (32768) +#define HAS_UNPREDICTABLE_JUMP_FLAG (65536) +#define HAS_NEEDS_GUARD_IP_FLAG (131072) +#define HAS_RECORDS_VALUE_FLAG (262144) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -1044,10 +1083,13 @@ enum InstructionFormat { #define OPCODE_HAS_ESCAPES(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ESCAPES_FLAG)) #define OPCODE_HAS_EXIT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EXIT_FLAG)) #define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG)) -#define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) -#define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) +#define OPCODE_HAS_SYNC_SP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_SYNC_SP_FLAG)) #define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) #define OPCODE_HAS_NO_SAVE_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NO_SAVE_IP_FLAG)) +#define OPCODE_HAS_PERIODIC(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PERIODIC_FLAG)) +#define OPCODE_HAS_UNPREDICTABLE_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_UNPREDICTABLE_JUMP_FLAG)) +#define OPCODE_HAS_NEEDS_GUARD_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NEEDS_GUARD_IP_FLAG)) +#define OPCODE_HAS_RECORDS_VALUE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_RECORDS_VALUE_FLAG)) #define OPARG_SIMPLE 0 #define OPARG_CACHE_1 1 @@ -1064,27 +1106,28 @@ enum InstructionFormat { struct opcode_metadata { uint8_t valid_entry; uint8_t instr_format; - uint16_t flags; + uint32_t flags; }; -extern const struct opcode_metadata _PyOpcode_opcode_metadata[267]; +PyAPI_DATA(const struct opcode_metadata) _PyOpcode_opcode_metadata[267]; #ifdef NEED_OPCODE_METADATA const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { - [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, - [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, + [BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, - [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG }, - [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_LIST_SLICE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_SUBSCR_LIST_SLICE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, + [BINARY_OP_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_SUBSCR_USTR_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_INTERPOLATION] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1096,34 +1139,36 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BUILD_TEMPLATE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [CACHE] = { true, INSTR_FMT_IX, 0 }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_EX_NON_PY_GENERAL] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_EX_PY] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LEN] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_LEN] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1131,9 +1176,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, - [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1143,9 +1188,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1153,30 +1198,32 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_VIRTUAL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [GET_ITER_SELF] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, + [GET_ITER_VIRTUAL] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_NOT_TAKEN] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_POP_ITER] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, @@ -1184,9 +1231,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1195,21 +1242,21 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [JUMP_BACKWARD_NO_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, @@ -1218,24 +1265,24 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_BORROW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SMALL_INT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, @@ -1250,23 +1297,26 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [POP_TOP] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, - [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, - [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [RESUME] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [RESUME_CHECK] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [RESUME_CHECK_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND_ASYNC_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [SEND_VIRTUAL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG }, [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG }, @@ -1275,26 +1325,27 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, - [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, - [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TRACE_RECORD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_NOT] = { true, INSTR_FMT_IX, 0 }, [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [ANNOTATIONS_PLACEHOLDER] = { true, -1, HAS_PURE_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1309,7 +1360,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { }; #endif -#define MAX_UOP_PER_EXPANSION 10 +#define MAX_UOP_PER_EXPANSION 11 struct opcode_macro_expansion { int nuops; struct { int16_t uop; int8_t size; int8_t offset; } uops[MAX_UOP_PER_EXPANSION]; @@ -1319,22 +1370,23 @@ extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256]; #ifdef NEED_OPCODE_METADATA const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { - [BINARY_OP] = { .nuops = 1, .uops = { { _BINARY_OP, OPARG_SIMPLE, 4 } } }, - [BINARY_OP_ADD_FLOAT] = { .nuops = 3, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_FLOAT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_ADD_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_ADD_UNICODE] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_UNICODE, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_EXTEND] = { .nuops = 2, .uops = { { _GUARD_BINARY_OP_EXTEND, 4, 1 }, { _BINARY_OP_EXTEND, 4, 1 } } }, + [BINARY_OP] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _BINARY_OP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, + [BINARY_OP_ADD_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_ADD_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_ADD_UNICODE] = { .nuops = 5, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_UNICODE, OPARG_SIMPLE, 5 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 5 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_EXTEND] = { .nuops = 4, .uops = { { _GUARD_BINARY_OP_EXTEND, 4, 1 }, { _BINARY_OP_EXTEND, 4, 1 }, { _POP_TOP, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, [BINARY_OP_INPLACE_ADD_UNICODE] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_INPLACE_ADD_UNICODE, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_MULTIPLY_FLOAT] = { .nuops = 3, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_MULTIPLY_FLOAT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_MULTIPLY_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_MULTIPLY_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_DICT] = { .nuops = 2, .uops = { { _GUARD_NOS_DICT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_DICT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_GETITEM] = { .nuops = 4, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 5 }, { _BINARY_OP_SUBSCR_CHECK_FUNC, OPARG_SIMPLE, 5 }, { _BINARY_OP_SUBSCR_INIT_CALL, OPARG_SIMPLE, 5 }, { _PUSH_FRAME, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_LIST_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_LIST_SLICE] = { .nuops = 3, .uops = { { _GUARD_TOS_SLICE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_SLICE, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_STR_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_STR_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_TUPLE_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_TUPLE, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_TUPLE_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 3, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBTRACT_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_INT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_MULTIPLY_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_MULTIPLY_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_MULTIPLY_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_MULTIPLY_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_DICT] = { .nuops = 5, .uops = { { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_DICT_SUBSCRIPT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_DICT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_GETITEM] = { .nuops = 5, .uops = { { _RECORD_NOS, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 5 }, { _BINARY_OP_SUBSCR_CHECK_FUNC, OPARG_SIMPLE, 5 }, { _BINARY_OP_SUBSCR_INIT_CALL, OPARG_SIMPLE, 5 }, { _PUSH_FRAME, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_LIST_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_LIST_SLICE] = { .nuops = 5, .uops = { { _GUARD_TOS_SLICE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_SLICE, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_STR_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_COMPACT_ASCII, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_STR_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_TUPLE_INT] = { .nuops = 6, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_TUPLE, OPARG_SIMPLE, 0 }, { _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_TUPLE_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_USTR_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_USTR_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBTRACT_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, [BINARY_SLICE] = { .nuops = 1, .uops = { { _BINARY_SLICE, OPARG_SIMPLE, 0 } } }, [BUILD_INTERPOLATION] = { .nuops = 1, .uops = { { _BUILD_INTERPOLATION, OPARG_SIMPLE, 0 } } }, [BUILD_LIST] = { .nuops = 1, .uops = { { _BUILD_LIST, OPARG_SIMPLE, 0 } } }, @@ -1344,40 +1396,42 @@ _PyOpcode_macro_expansion[256] = { [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, OPARG_SIMPLE, 0 } } }, [BUILD_TEMPLATE] = { .nuops = 1, .uops = { { _BUILD_TEMPLATE, OPARG_SIMPLE, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, OPARG_SIMPLE, 0 } } }, - [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 4, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_AND_ALLOCATE_OBJECT, 2, 1 }, { _CREATE_INIT_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 10, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_BOUND_METHOD_GENERAL] = { .nuops = 7, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_FAST] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_O] = { .nuops = 2, .uops = { { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_1, OPARG_SIMPLE, 0 } } }, - [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, OPARG_SIMPLE, 0 } } }, + [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_OBJECT, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _ALLOCATE_OBJECT, OPARG_SIMPLE, 3 }, { _CREATE_INIT_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 11, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_BOUND_METHOD_GENERAL] = { .nuops = 8, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_BUILTIN_CLASS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_FAST] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_O] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_EX_NON_PY_GENERAL] = { .nuops = 4, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CALL_FUNCTION_EX_NON_PY_GENERAL, OPARG_SIMPLE, 1 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 1 } } }, + [CALL_EX_PY] = { .nuops = 7, .uops = { { _RECORD_4OS, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CHECK_IS_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _PY_FRAME_EX, OPARG_SIMPLE, 1 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [CALL_INTRINSIC_1] = { .nuops = 2, .uops = { { _CALL_INTRINSIC_1, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [CALL_INTRINSIC_2] = { .nuops = 3, .uops = { { _CALL_INTRINSIC_2, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 3, .uops = { { _GUARD_THIRD_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_ISINSTANCE, OPARG_SIMPLE, 3 }, { _CALL_ISINSTANCE, OPARG_SIMPLE, 3 } } }, - [CALL_KW_BOUND_METHOD] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, OPARG_SIMPLE, 3 }, { _CALL_KW_NON_PY, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_KW_PY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_LEN] = { .nuops = 3, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_LEN, OPARG_SIMPLE, 3 }, { _CALL_LEN, OPARG_SIMPLE, 3 } } }, - [CALL_LIST_APPEND] = { .nuops = 4, .uops = { { _GUARD_CALLABLE_LIST_APPEND, OPARG_SIMPLE, 3 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 3 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 3 }, { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_NON_PY_GENERAL] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_PY_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_PY_GENERAL] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_STR_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_STR_1, OPARG_SIMPLE, 3 }, { _CALL_STR_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_TUPLE_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TUPLE_1, OPARG_SIMPLE, 3 }, { _CALL_TUPLE_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_TYPE_1] = { .nuops = 3, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TYPE_1, OPARG_SIMPLE, 3 }, { _CALL_TYPE_1, OPARG_SIMPLE, 3 } } }, + [CALL_KW_BOUND_METHOD] = { .nuops = 7, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, OPARG_SIMPLE, 3 }, { _CALL_KW_NON_PY, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_KW_PY] = { .nuops = 7, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_LEN] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_LEN, OPARG_SIMPLE, 3 }, { _CALL_LEN, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, + [CALL_LIST_APPEND] = { .nuops = 6, .uops = { { _GUARD_CALLABLE_LIST_APPEND, OPARG_SIMPLE, 3 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 3 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 3 }, { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, + [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 5, .uops = { { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 8, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_NON_PY_GENERAL] = { .nuops = 4, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_PY_EXACT_ARGS] = { .nuops = 9, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_PY_GENERAL] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_STR_1] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_STR_1, OPARG_SIMPLE, 3 }, { _CALL_STR_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_TUPLE_1] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TUPLE_1, OPARG_SIMPLE, 3 }, { _CALL_TUPLE_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_TYPE_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TYPE_1, OPARG_SIMPLE, 3 }, { _CALL_TYPE_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { _CHECK_EG_MATCH, OPARG_SIMPLE, 0 } } }, [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { _CHECK_EXC_MATCH, OPARG_SIMPLE, 0 } } }, [COMPARE_OP] = { .nuops = 1, .uops = { { _COMPARE_OP, OPARG_SIMPLE, 0 } } }, - [COMPARE_OP_FLOAT] = { .nuops = 3, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _COMPARE_OP_FLOAT, OPARG_SIMPLE, 1 } } }, - [COMPARE_OP_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _COMPARE_OP_INT, OPARG_SIMPLE, 1 } } }, - [COMPARE_OP_STR] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _COMPARE_OP_STR, OPARG_SIMPLE, 1 } } }, - [CONTAINS_OP] = { .nuops = 1, .uops = { { _CONTAINS_OP, OPARG_SIMPLE, 0 } } }, - [CONTAINS_OP_DICT] = { .nuops = 2, .uops = { { _GUARD_TOS_DICT, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_DICT, OPARG_SIMPLE, 1 } } }, - [CONTAINS_OP_SET] = { .nuops = 2, .uops = { { _GUARD_TOS_ANY_SET, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_SET, OPARG_SIMPLE, 1 } } }, + [COMPARE_OP_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _COMPARE_OP_FLOAT, OPARG_SIMPLE, 1 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 1 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 1 } } }, + [COMPARE_OP_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _COMPARE_OP_INT, OPARG_SIMPLE, 1 }, { _POP_TOP_INT, OPARG_SIMPLE, 1 }, { _POP_TOP_INT, OPARG_SIMPLE, 1 } } }, + [COMPARE_OP_STR] = { .nuops = 5, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _COMPARE_OP_STR, OPARG_SIMPLE, 1 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 1 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 1 } } }, + [CONTAINS_OP] = { .nuops = 3, .uops = { { _CONTAINS_OP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [CONTAINS_OP_DICT] = { .nuops = 4, .uops = { { _GUARD_TOS_ANY_DICT, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_DICT, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, + [CONTAINS_OP_SET] = { .nuops = 4, .uops = { { _GUARD_TOS_ANY_SET, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_SET, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, [CONVERT_VALUE] = { .nuops = 1, .uops = { { _CONVERT_VALUE, OPARG_SIMPLE, 0 } } }, [COPY] = { .nuops = 1, .uops = { { _COPY, OPARG_SIMPLE, 0 } } }, [COPY_FREE_VARS] = { .nuops = 1, .uops = { { _COPY_FREE_VARS, OPARG_SIMPLE, 0 } } }, @@ -1387,42 +1441,48 @@ _PyOpcode_macro_expansion[256] = { [DELETE_GLOBAL] = { .nuops = 1, .uops = { { _DELETE_GLOBAL, OPARG_SIMPLE, 0 } } }, [DELETE_NAME] = { .nuops = 1, .uops = { { _DELETE_NAME, OPARG_SIMPLE, 0 } } }, [DELETE_SUBSCR] = { .nuops = 1, .uops = { { _DELETE_SUBSCR, OPARG_SIMPLE, 0 } } }, - [DICT_MERGE] = { .nuops = 1, .uops = { { _DICT_MERGE, OPARG_SIMPLE, 0 } } }, - [DICT_UPDATE] = { .nuops = 1, .uops = { { _DICT_UPDATE, OPARG_SIMPLE, 0 } } }, + [DICT_MERGE] = { .nuops = 2, .uops = { { _DICT_MERGE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [DICT_UPDATE] = { .nuops = 2, .uops = { { _DICT_UPDATE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [END_FOR] = { .nuops = 1, .uops = { { _END_FOR, OPARG_SIMPLE, 0 } } }, [END_SEND] = { .nuops = 1, .uops = { { _END_SEND, OPARG_SIMPLE, 0 } } }, [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { _EXIT_INIT_CHECK, OPARG_SIMPLE, 0 } } }, [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { _FORMAT_SIMPLE, OPARG_SIMPLE, 0 } } }, [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { _FORMAT_WITH_SPEC, OPARG_SIMPLE, 0 } } }, - [FOR_ITER] = { .nuops = 1, .uops = { { _FOR_ITER, OPARG_REPLACED, 0 } } }, - [FOR_ITER_GEN] = { .nuops = 3, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _FOR_ITER_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [FOR_ITER] = { .nuops = 2, .uops = { { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _FOR_ITER, OPARG_REPLACED, 0 } } }, + [FOR_ITER_GEN] = { .nuops = 4, .uops = { { _RECORD_NOS_GEN_FUNC, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _FOR_ITER_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, OPARG_SIMPLE, 1 }, { _ITER_JUMP_LIST, OPARG_REPLACED, 1 }, { _ITER_NEXT_LIST, OPARG_REPLACED, 1 } } }, [FOR_ITER_RANGE] = { .nuops = 3, .uops = { { _ITER_CHECK_RANGE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_RANGE, OPARG_REPLACED, 1 }, { _ITER_NEXT_RANGE, OPARG_SIMPLE, 1 } } }, [FOR_ITER_TUPLE] = { .nuops = 3, .uops = { { _ITER_CHECK_TUPLE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_TUPLE, OPARG_REPLACED, 1 }, { _ITER_NEXT_TUPLE, OPARG_SIMPLE, 1 } } }, + [FOR_ITER_VIRTUAL] = { .nuops = 2, .uops = { { _GUARD_TOS_NOT_NULL, OPARG_SIMPLE, 1 }, { _FOR_ITER_VIRTUAL, OPARG_REPLACED, 1 } } }, [GET_AITER] = { .nuops = 1, .uops = { { _GET_AITER, OPARG_SIMPLE, 0 } } }, [GET_ANEXT] = { .nuops = 1, .uops = { { _GET_ANEXT, OPARG_SIMPLE, 0 } } }, [GET_AWAITABLE] = { .nuops = 1, .uops = { { _GET_AWAITABLE, OPARG_SIMPLE, 0 } } }, - [GET_ITER] = { .nuops = 1, .uops = { { _GET_ITER, OPARG_SIMPLE, 0 } } }, + [GET_ITER] = { .nuops = 2, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _GET_ITER, OPARG_SIMPLE, 0 } } }, + [GET_ITER_SELF] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _GUARD_ITERATOR, OPARG_SIMPLE, 1 }, { _PUSH_NULL, OPARG_SIMPLE, 1 } } }, + [GET_ITER_VIRTUAL] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _GUARD_ITER_VIRTUAL, OPARG_SIMPLE, 1 }, { _PUSH_TAGGED_ZERO, OPARG_SIMPLE, 1 } } }, [GET_LEN] = { .nuops = 1, .uops = { { _GET_LEN, OPARG_SIMPLE, 0 } } }, - [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { _GET_YIELD_FROM_ITER, OPARG_SIMPLE, 0 } } }, [IMPORT_FROM] = { .nuops = 1, .uops = { { _IMPORT_FROM, OPARG_SIMPLE, 0 } } }, [IMPORT_NAME] = { .nuops = 1, .uops = { { _IMPORT_NAME, OPARG_SIMPLE, 0 } } }, - [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, OPARG_SIMPLE, 0 } } }, + [IS_OP] = { .nuops = 3, .uops = { { _IS_OP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [JUMP_BACKWARD] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, + [JUMP_BACKWARD_NO_INTERRUPT] = { .nuops = 1, .uops = { { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 0 } } }, + [JUMP_BACKWARD_NO_JIT] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, OPARG_SIMPLE, 0 } } }, - [LIST_EXTEND] = { .nuops = 1, .uops = { { _LIST_EXTEND, OPARG_SIMPLE, 0 } } }, + [LIST_EXTEND] = { .nuops = 2, .uops = { { _LIST_EXTEND, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, OPARG_SIMPLE, 8 } } }, - [LOAD_ATTR_CLASS] = { .nuops = 3, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { .nuops = 4, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _GUARD_TYPE_VERSION, 2, 3 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, - [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, OPARG_SIMPLE, 3 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, - [LOAD_ATTR_MODULE] = { .nuops = 3, .uops = { { _LOAD_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, OPERAND1_1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, OPARG_SIMPLE, 3 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, - [LOAD_ATTR_PROPERTY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_PROPERTY_FRAME, 4, 5 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 9 }, { _PUSH_FRAME, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_SLOT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_WITH_HINT, 1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_CLASS] = { .nuops = 4, .uops = { { _RECORD_TOS, OPARG_SIMPLE, 1 }, { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { .nuops = 5, .uops = { { _RECORD_TOS, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_CLASS, 2, 3 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { .nuops = 7, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME, 2, 3 }, { _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME, OPERAND1_4, 5 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 9 }, { _PUSH_FRAME, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 6, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, + [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_MODULE] = { .nuops = 4, .uops = { { _LOAD_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, OPERAND1_1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_PROPERTY] = { .nuops = 7, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_PROPERTY_FRAME, 2, 3 }, { _LOAD_ATTR_PROPERTY_FRAME, OPERAND1_4, 5 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 9 }, { _PUSH_FRAME, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_SLOT] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_WITH_HINT] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_WITH_HINT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { _LOAD_BUILD_CLASS, OPARG_SIMPLE, 0 } } }, [LOAD_COMMON_CONSTANT] = { .nuops = 1, .uops = { { _LOAD_COMMON_CONSTANT, OPARG_SIMPLE, 0 } } }, [LOAD_CONST] = { .nuops = 1, .uops = { { _LOAD_CONST, OPARG_SIMPLE, 0 } } }, @@ -1440,13 +1500,13 @@ _PyOpcode_macro_expansion[256] = { [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, OPARG_SIMPLE, 0 } } }, [LOAD_NAME] = { .nuops = 1, .uops = { { _LOAD_NAME, OPARG_SIMPLE, 0 } } }, [LOAD_SMALL_INT] = { .nuops = 1, .uops = { { _LOAD_SMALL_INT, OPARG_SIMPLE, 0 } } }, - [LOAD_SPECIAL] = { .nuops = 2, .uops = { { _INSERT_NULL, OPARG_SIMPLE, 0 }, { _LOAD_SPECIAL, OPARG_SIMPLE, 0 } } }, + [LOAD_SPECIAL] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _INSERT_NULL, OPARG_SIMPLE, 0 }, { _LOAD_SPECIAL, OPARG_SIMPLE, 0 } } }, [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_ATTR, OPARG_SIMPLE, 1 } } }, - [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_METHOD, OPARG_SIMPLE, 1 } } }, + [LOAD_SUPER_ATTR_METHOD] = { .nuops = 3, .uops = { { _RECORD_NOS, OPARG_SIMPLE, 0 }, { _GUARD_LOAD_SUPER_ATTR_METHOD, OPARG_SIMPLE, 1 }, { _LOAD_SUPER_ATTR_METHOD, OPARG_SIMPLE, 1 } } }, [MAKE_CELL] = { .nuops = 1, .uops = { { _MAKE_CELL, OPARG_SIMPLE, 0 } } }, - [MAKE_FUNCTION] = { .nuops = 1, .uops = { { _MAKE_FUNCTION, OPARG_SIMPLE, 0 } } }, + [MAKE_FUNCTION] = { .nuops = 2, .uops = { { _MAKE_FUNCTION, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [MAP_ADD] = { .nuops = 1, .uops = { { _MAP_ADD, OPARG_SIMPLE, 0 } } }, - [MATCH_CLASS] = { .nuops = 1, .uops = { { _MATCH_CLASS, OPARG_SIMPLE, 0 } } }, + [MATCH_CLASS] = { .nuops = 4, .uops = { { _MATCH_CLASS, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [MATCH_KEYS] = { .nuops = 1, .uops = { { _MATCH_KEYS, OPARG_SIMPLE, 0 } } }, [MATCH_MAPPING] = { .nuops = 1, .uops = { { _MATCH_MAPPING, OPARG_SIMPLE, 0 } } }, [MATCH_SEQUENCE] = { .nuops = 1, .uops = { { _MATCH_SEQUENCE, OPARG_SIMPLE, 0 } } }, @@ -1461,38 +1521,40 @@ _PyOpcode_macro_expansion[256] = { [POP_TOP] = { .nuops = 1, .uops = { { _POP_TOP, OPARG_SIMPLE, 0 } } }, [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { _PUSH_EXC_INFO, OPARG_SIMPLE, 0 } } }, [PUSH_NULL] = { .nuops = 1, .uops = { { _PUSH_NULL, OPARG_SIMPLE, 0 } } }, - [RESUME_CHECK] = { .nuops = 1, .uops = { { _RESUME_CHECK, OPARG_SIMPLE, 0 } } }, + [RESUME_CHECK] = { .nuops = 1, .uops = { { _RESUME_CHECK, OPARG_SIMPLE, 1 } } }, [RETURN_GENERATOR] = { .nuops = 1, .uops = { { _RETURN_GENERATOR, OPARG_SIMPLE, 0 } } }, - [RETURN_VALUE] = { .nuops = 1, .uops = { { _RETURN_VALUE, OPARG_SIMPLE, 0 } } }, - [SEND_GEN] = { .nuops = 3, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _SEND_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [RETURN_VALUE] = { .nuops = 2, .uops = { { _MAKE_HEAP_SAFE, OPARG_SIMPLE, 0 }, { _RETURN_VALUE, OPARG_SIMPLE, 0 } } }, + [SEND_ASYNC_GEN] = { .nuops = 2, .uops = { { _GUARD_3OS_ASYNC_GEN_ASEND, OPARG_SIMPLE, 1 }, { _SEND_ASYNC_GEN, OPARG_REPLACED, 1 } } }, + [SEND_GEN] = { .nuops = 4, .uops = { { _RECORD_3OS_GEN_FUNC, OPARG_SIMPLE, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _SEND_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [SEND_VIRTUAL] = { .nuops = 3, .uops = { { _GUARD_TOS_IS_NONE, OPARG_SIMPLE, 1 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 1 }, { _SEND_VIRTUAL, OPARG_REPLACED, 1 } } }, [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { _SETUP_ANNOTATIONS, OPARG_SIMPLE, 0 } } }, [SET_ADD] = { .nuops = 1, .uops = { { _SET_ADD, OPARG_SIMPLE, 0 } } }, [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, OPARG_SIMPLE, 0 } } }, - [SET_UPDATE] = { .nuops = 1, .uops = { { _SET_UPDATE, OPARG_SIMPLE, 0 } } }, + [SET_UPDATE] = { .nuops = 2, .uops = { { _SET_UPDATE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, OPARG_SIMPLE, 3 } } }, - [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION_AND_LOCK, 2, 1 }, { _GUARD_DORV_NO_DICT, OPARG_SIMPLE, 3 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, - [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, - [STORE_ATTR_WITH_HINT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_WITH_HINT, 1, 3 } } }, + [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 6, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _LOCK_OBJECT, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION_LOCKED, 2, 1 }, { _GUARD_DORV_NO_DICT, OPARG_SIMPLE, 3 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, + [STORE_ATTR_SLOT] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, + [STORE_ATTR_WITH_HINT] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_WITH_HINT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, [STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, OPARG_SIMPLE, 0 } } }, - [STORE_FAST] = { .nuops = 1, .uops = { { _STORE_FAST, OPARG_SIMPLE, 0 } } }, - [STORE_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _STORE_FAST, OPARG_TOP, 0 }, { _LOAD_FAST, OPARG_BOTTOM, 0 } } }, - [STORE_FAST_STORE_FAST] = { .nuops = 2, .uops = { { _STORE_FAST, OPARG_TOP, 0 }, { _STORE_FAST, OPARG_BOTTOM, 0 } } }, + [STORE_FAST] = { .nuops = 2, .uops = { { _SWAP_FAST, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [STORE_FAST_LOAD_FAST] = { .nuops = 3, .uops = { { _SWAP_FAST, OPARG_TOP, 0 }, { _POP_TOP, OPARG_TOP, 0 }, { _LOAD_FAST, OPARG_BOTTOM, 0 } } }, + [STORE_FAST_STORE_FAST] = { .nuops = 4, .uops = { { _SWAP_FAST, OPARG_TOP, 0 }, { _POP_TOP, OPARG_TOP, 0 }, { _SWAP_FAST, OPARG_BOTTOM, 0 }, { _POP_TOP, OPARG_BOTTOM, 0 } } }, [STORE_GLOBAL] = { .nuops = 1, .uops = { { _STORE_GLOBAL, OPARG_SIMPLE, 0 } } }, [STORE_NAME] = { .nuops = 1, .uops = { { _STORE_NAME, OPARG_SIMPLE, 0 } } }, [STORE_SLICE] = { .nuops = 1, .uops = { { _STORE_SLICE, OPARG_SIMPLE, 0 } } }, [STORE_SUBSCR] = { .nuops = 1, .uops = { { _STORE_SUBSCR, OPARG_SIMPLE, 0 } } }, - [STORE_SUBSCR_DICT] = { .nuops = 2, .uops = { { _GUARD_NOS_DICT, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_DICT, OPARG_SIMPLE, 1 } } }, - [STORE_SUBSCR_LIST_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_LIST_INT, OPARG_SIMPLE, 1 } } }, + [STORE_SUBSCR_DICT] = { .nuops = 4, .uops = { { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_DICT_STORE_SUBSCRIPT, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_DICT, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, + [STORE_SUBSCR_LIST_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_LIST_INT, OPARG_SIMPLE, 1 }, { _POP_TOP_INT, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, [SWAP] = { .nuops = 1, .uops = { { _SWAP, OPARG_SIMPLE, 0 } } }, [TO_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL, OPARG_SIMPLE, 2 } } }, - [TO_BOOL_ALWAYS_TRUE] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _REPLACE_WITH_TRUE, OPARG_SIMPLE, 3 } } }, + [TO_BOOL_ALWAYS_TRUE] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _REPLACE_WITH_TRUE, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL_BOOL, OPARG_SIMPLE, 3 } } }, - [TO_BOOL_INT] = { .nuops = 1, .uops = { { _TO_BOOL_INT, OPARG_SIMPLE, 3 } } }, - [TO_BOOL_LIST] = { .nuops = 2, .uops = { { _GUARD_TOS_LIST, OPARG_SIMPLE, 0 }, { _TO_BOOL_LIST, OPARG_SIMPLE, 3 } } }, + [TO_BOOL_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _TO_BOOL_INT, OPARG_SIMPLE, 3 }, { _POP_TOP_INT, OPARG_SIMPLE, 3 } } }, + [TO_BOOL_LIST] = { .nuops = 3, .uops = { { _GUARD_TOS_LIST, OPARG_SIMPLE, 0 }, { _TO_BOOL_LIST, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [TO_BOOL_NONE] = { .nuops = 1, .uops = { { _TO_BOOL_NONE, OPARG_SIMPLE, 3 } } }, - [TO_BOOL_STR] = { .nuops = 2, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _TO_BOOL_STR, OPARG_SIMPLE, 3 } } }, - [UNARY_INVERT] = { .nuops = 1, .uops = { { _UNARY_INVERT, OPARG_SIMPLE, 0 } } }, - [UNARY_NEGATIVE] = { .nuops = 1, .uops = { { _UNARY_NEGATIVE, OPARG_SIMPLE, 0 } } }, + [TO_BOOL_STR] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _TO_BOOL_STR, OPARG_SIMPLE, 3 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 3 } } }, + [UNARY_INVERT] = { .nuops = 2, .uops = { { _UNARY_INVERT, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [UNARY_NEGATIVE] = { .nuops = 2, .uops = { { _UNARY_NEGATIVE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [UNARY_NOT] = { .nuops = 1, .uops = { { _UNARY_NOT, OPARG_SIMPLE, 0 } } }, [UNPACK_EX] = { .nuops = 1, .uops = { { _UNPACK_EX, OPARG_SIMPLE, 0 } } }, [UNPACK_SEQUENCE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE, OPARG_SIMPLE, 0 } } }, @@ -1500,11 +1562,11 @@ _PyOpcode_macro_expansion[256] = { [UNPACK_SEQUENCE_TUPLE] = { .nuops = 2, .uops = { { _GUARD_TOS_TUPLE, OPARG_SIMPLE, 0 }, { _UNPACK_SEQUENCE_TUPLE, OPARG_SIMPLE, 1 } } }, [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 2, .uops = { { _GUARD_TOS_TUPLE, OPARG_SIMPLE, 0 }, { _UNPACK_SEQUENCE_TWO_TUPLE, OPARG_SIMPLE, 1 } } }, [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { _WITH_EXCEPT_START, OPARG_SIMPLE, 0 } } }, - [YIELD_VALUE] = { .nuops = 1, .uops = { { _YIELD_VALUE, OPARG_SIMPLE, 0 } } }, + [YIELD_VALUE] = { .nuops = 2, .uops = { { _MAKE_HEAP_SAFE, OPARG_SIMPLE, 0 }, { _YIELD_VALUE, OPARG_SIMPLE, 0 } } }, }; #endif // NEED_OPCODE_METADATA -extern const char *_PyOpcode_OpName[267]; +PyAPI_DATA(const char) *_PyOpcode_OpName[267]; #ifdef NEED_OPCODE_METADATA const char *_PyOpcode_OpName[267] = { [ANNOTATIONS_PLACEHOLDER] = "ANNOTATIONS_PLACEHOLDER", @@ -1522,6 +1584,7 @@ const char *_PyOpcode_OpName[267] = { [BINARY_OP_SUBSCR_LIST_SLICE] = "BINARY_OP_SUBSCR_LIST_SLICE", [BINARY_OP_SUBSCR_STR_INT] = "BINARY_OP_SUBSCR_STR_INT", [BINARY_OP_SUBSCR_TUPLE_INT] = "BINARY_OP_SUBSCR_TUPLE_INT", + [BINARY_OP_SUBSCR_USTR_INT] = "BINARY_OP_SUBSCR_USTR_INT", [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", [BINARY_SLICE] = "BINARY_SLICE", @@ -1542,6 +1605,8 @@ const char *_PyOpcode_OpName[267] = { [CALL_BUILTIN_FAST] = "CALL_BUILTIN_FAST", [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [CALL_BUILTIN_O] = "CALL_BUILTIN_O", + [CALL_EX_NON_PY_GENERAL] = "CALL_EX_NON_PY_GENERAL", + [CALL_EX_PY] = "CALL_EX_PY", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", [CALL_INTRINSIC_1] = "CALL_INTRINSIC_1", [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", @@ -1596,12 +1661,14 @@ const char *_PyOpcode_OpName[267] = { [FOR_ITER_LIST] = "FOR_ITER_LIST", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", + [FOR_ITER_VIRTUAL] = "FOR_ITER_VIRTUAL", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", [GET_AWAITABLE] = "GET_AWAITABLE", [GET_ITER] = "GET_ITER", + [GET_ITER_SELF] = "GET_ITER_SELF", + [GET_ITER_VIRTUAL] = "GET_ITER_VIRTUAL", [GET_LEN] = "GET_LEN", - [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [IMPORT_FROM] = "IMPORT_FROM", [IMPORT_NAME] = "IMPORT_NAME", [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", @@ -1699,10 +1766,13 @@ const char *_PyOpcode_OpName[267] = { [RESERVED] = "RESERVED", [RESUME] = "RESUME", [RESUME_CHECK] = "RESUME_CHECK", + [RESUME_CHECK_JIT] = "RESUME_CHECK_JIT", [RETURN_GENERATOR] = "RETURN_GENERATOR", [RETURN_VALUE] = "RETURN_VALUE", [SEND] = "SEND", + [SEND_ASYNC_GEN] = "SEND_ASYNC_GEN", [SEND_GEN] = "SEND_GEN", + [SEND_VIRTUAL] = "SEND_VIRTUAL", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", [SETUP_CLEANUP] = "SETUP_CLEANUP", [SETUP_FINALLY] = "SETUP_FINALLY", @@ -1733,6 +1803,7 @@ const char *_PyOpcode_OpName[267] = { [TO_BOOL_LIST] = "TO_BOOL_LIST", [TO_BOOL_NONE] = "TO_BOOL_NONE", [TO_BOOL_STR] = "TO_BOOL_STR", + [TRACE_RECORD] = "TRACE_RECORD", [UNARY_INVERT] = "UNARY_INVERT", [UNARY_NEGATIVE] = "UNARY_NEGATIVE", [UNARY_NOT] = "UNARY_NOT", @@ -1746,12 +1817,14 @@ const char *_PyOpcode_OpName[267] = { }; #endif -extern const uint8_t _PyOpcode_Caches[256]; +PyAPI_DATA(const uint8_t) _PyOpcode_Caches[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Caches[256] = { + [RESUME] = 1, [TO_BOOL] = 3, [STORE_SUBSCR] = 1, [SEND] = 1, + [FOR_ITER] = 1, [UNPACK_SEQUENCE] = 1, [STORE_ATTR] = 4, [LOAD_GLOBAL] = 4, @@ -1764,16 +1837,18 @@ const uint8_t _PyOpcode_Caches[256] = { [POP_JUMP_IF_FALSE] = 1, [POP_JUMP_IF_NONE] = 1, [POP_JUMP_IF_NOT_NONE] = 1, - [FOR_ITER] = 1, + [GET_ITER] = 1, [CALL] = 3, [CALL_KW] = 3, + [CALL_FUNCTION_EX] = 1, [BINARY_OP] = 5, }; #endif -extern const uint8_t _PyOpcode_Deopt[256]; +PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { + [120] = 120, [121] = 121, [122] = 122, [123] = 123, @@ -1781,15 +1856,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [125] = 125, [126] = 126, [127] = 127, - [210] = 210, - [211] = 211, - [212] = 212, - [213] = 213, - [214] = 214, - [215] = 215, - [216] = 216, - [217] = 217, - [218] = 218, [219] = 219, [220] = 220, [221] = 221, @@ -1804,7 +1870,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [230] = 230, [231] = 231, [232] = 232, - [233] = 233, [BINARY_OP] = BINARY_OP, [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, @@ -1819,6 +1884,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [BINARY_OP_SUBSCR_LIST_SLICE] = BINARY_OP, [BINARY_OP_SUBSCR_STR_INT] = BINARY_OP, [BINARY_OP_SUBSCR_TUPLE_INT] = BINARY_OP, + [BINARY_OP_SUBSCR_USTR_INT] = BINARY_OP, [BINARY_OP_SUBTRACT_FLOAT] = BINARY_OP, [BINARY_OP_SUBTRACT_INT] = BINARY_OP, [BINARY_SLICE] = BINARY_SLICE, @@ -1839,6 +1905,8 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_BUILTIN_FAST] = CALL, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, [CALL_BUILTIN_O] = CALL, + [CALL_EX_NON_PY_GENERAL] = CALL_FUNCTION_EX, + [CALL_EX_PY] = CALL_FUNCTION_EX, [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, [CALL_INTRINSIC_1] = CALL_INTRINSIC_1, [CALL_INTRINSIC_2] = CALL_INTRINSIC_2, @@ -1893,12 +1961,14 @@ const uint8_t _PyOpcode_Deopt[256] = { [FOR_ITER_LIST] = FOR_ITER, [FOR_ITER_RANGE] = FOR_ITER, [FOR_ITER_TUPLE] = FOR_ITER, + [FOR_ITER_VIRTUAL] = FOR_ITER, [GET_AITER] = GET_AITER, [GET_ANEXT] = GET_ANEXT, [GET_AWAITABLE] = GET_AWAITABLE, [GET_ITER] = GET_ITER, + [GET_ITER_SELF] = GET_ITER, + [GET_ITER_VIRTUAL] = GET_ITER, [GET_LEN] = GET_LEN, - [GET_YIELD_FROM_ITER] = GET_YIELD_FROM_ITER, [IMPORT_FROM] = IMPORT_FROM, [IMPORT_NAME] = IMPORT_NAME, [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, @@ -1990,10 +2060,13 @@ const uint8_t _PyOpcode_Deopt[256] = { [RESERVED] = RESERVED, [RESUME] = RESUME, [RESUME_CHECK] = RESUME, + [RESUME_CHECK_JIT] = RESUME, [RETURN_GENERATOR] = RETURN_GENERATOR, [RETURN_VALUE] = RETURN_VALUE, [SEND] = SEND, + [SEND_ASYNC_GEN] = SEND, [SEND_GEN] = SEND, + [SEND_VIRTUAL] = SEND, [SETUP_ANNOTATIONS] = SETUP_ANNOTATIONS, [SET_ADD] = SET_ADD, [SET_FUNCTION_ATTRIBUTE] = SET_FUNCTION_ATTRIBUTE, @@ -2020,6 +2093,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [TO_BOOL_LIST] = TO_BOOL, [TO_BOOL_NONE] = TO_BOOL, [TO_BOOL_STR] = TO_BOOL, + [TRACE_RECORD] = TRACE_RECORD, [UNARY_INVERT] = UNARY_INVERT, [UNARY_NEGATIVE] = UNARY_NEGATIVE, [UNARY_NOT] = UNARY_NOT, @@ -2035,6 +2109,7 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ + case 120: \ case 121: \ case 122: \ case 123: \ @@ -2042,15 +2117,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 125: \ case 126: \ case 127: \ - case 210: \ - case 211: \ - case 212: \ - case 213: \ - case 214: \ - case 215: \ - case 216: \ - case 217: \ - case 218: \ case 219: \ case 220: \ case 221: \ @@ -2065,7 +2131,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 230: \ case 231: \ case 232: \ - case 233: \ ; struct pseudo_targets { uint8_t as_sequence; diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index 79a1a242556a52b..7067b48ec22cb3c 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -73,16 +73,30 @@ extern "C" { #define CONSTANT_BUILTIN_TUPLE 2 #define CONSTANT_BUILTIN_ALL 3 #define CONSTANT_BUILTIN_ANY 4 -#define NUM_COMMON_CONSTANTS 5 +#define CONSTANT_BUILTIN_LIST 5 +#define CONSTANT_BUILTIN_SET 6 +#define CONSTANT_NONE 7 +#define CONSTANT_EMPTY_STR 8 +#define CONSTANT_TRUE 9 +#define CONSTANT_FALSE 10 +#define CONSTANT_MINUS_ONE 11 +#define CONSTANT_BUILTIN_FROZENSET 12 +#define CONSTANT_EMPTY_TUPLE 13 +#define NUM_COMMON_CONSTANTS 14 /* Values used in the oparg for RESUME */ #define RESUME_AT_FUNC_START 0 #define RESUME_AFTER_YIELD 1 #define RESUME_AFTER_YIELD_FROM 2 #define RESUME_AFTER_AWAIT 3 +#define RESUME_AT_GEN_EXPR_START 4 -#define RESUME_OPARG_LOCATION_MASK 0x3 -#define RESUME_OPARG_DEPTH1_MASK 0x4 +#define RESUME_OPARG_LOCATION_MASK 0x7 +#define RESUME_OPARG_DEPTH1_MASK 0x8 + +#define GET_ITER_YIELD_FROM 1 +#define GET_ITER_YIELD_FROM_NO_CHECK 2 +#define GET_ITER_YIELD_FROM_CORO_CHECK 3 #ifdef __cplusplus } diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 8b7f12bf03d6242..3d60638649dcb5a 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -9,66 +9,183 @@ extern "C" { #endif #include "pycore_typedefs.h" // _PyInterpreterFrame +#include "pycore_jit_publish.h" +#include "pycore_uop.h" // _PyUOpInstruction #include "pycore_uop_ids.h" #include "pycore_stackref.h" // _PyStackRef +#include "pycore_optimizer_types.h" #include +/* Fitness controls how long a trace can grow. + * Starts at FITNESS_INITIAL, then decreases from per-bytecode buffer usage + * plus branch/frame heuristics. The trace stops when fitness drops below the + * current exit_quality. + * + * Design targets for the constants below: + * 1. Reaching the abstract frame-depth limit should drop fitness below + * EXIT_QUALITY_SPECIALIZABLE. + * 2. A backward edge should leave budget for roughly N_BACKWARD_SLACK more + * bytecodes, assuming AVG_SLOTS_PER_INSTRUCTION. + * 3. Roughly seven balanced branches should reduce fitness to + * EXIT_QUALITY_DEFAULT after per-slot costs. + * 4. A push followed by a matching return is net-zero on frame-specific + * fitness, excluding per-slot costs. + */ +#define OPTIMIZER_EFFECTIVENESS 2 +#define MAX_TARGET_LENGTH (FITNESS_INITIAL / OPTIMIZER_EFFECTIVENESS) + +/* Exit quality thresholds: trace stops when fitness < exit_quality. + * Higher = trace is more willing to stop here. */ +#define EXIT_QUALITY_CLOSE_LOOP (FITNESS_INITIAL - AVG_SLOTS_PER_INSTRUCTION*4) +#define EXIT_QUALITY_ENTER_EXECUTOR (FITNESS_INITIAL * 1 / 8) +#define EXIT_QUALITY_DEFAULT (FITNESS_INITIAL / 40) +#define EXIT_QUALITY_SPECIALIZABLE (FITNESS_INITIAL / 80) + +/* Estimated buffer slots per bytecode, used only to derive heuristics. + * Runtime charging uses trace-buffer capacity consumed for each bytecode. */ +#define AVG_SLOTS_PER_INSTRUCTION 6 + +/* Heuristic backward-edge exit quality: leave room for about 1 unroll and + * N_BACKWARD_SLACK more bytecodes before reaching EXIT_QUALITY_CLOSE_LOOP, + * based on AVG_SLOTS_PER_INSTRUCTION. */ +#define N_BACKWARD_SLACK 10 +#define EXIT_QUALITY_BACKWARD_EDGE (EXIT_QUALITY_CLOSE_LOOP / 2 - N_BACKWARD_SLACK * AVG_SLOTS_PER_INSTRUCTION) + +/* Penalty for a balanced branch. + * It is sized so repeated balanced branches can drive a trace toward + * EXIT_QUALITY_DEFAULT, while compute_branch_penalty() keeps any single branch + * from dominating the budget. + */ +#define FITNESS_BRANCH_BALANCED ((FITNESS_INITIAL - EXIT_QUALITY_DEFAULT - \ + (MAX_TARGET_LENGTH / 14 * AVG_SLOTS_PER_INSTRUCTION)) / (14)) + + +typedef struct _PyJitUopBuffer { + _PyUOpInstruction *start; + _PyUOpInstruction *next; + _PyUOpInstruction *end; +} _PyJitUopBuffer; + +typedef struct _JitOptRefBuffer { + JitOptRef *used; + JitOptRef *end; +} _JitOptRefBuffer; + +typedef struct _JitOptContext { + char done; + char out_of_space; + bool contradiction; + // Has the builtins dict been watched? + bool builtins_watched; + // The current "executing" frame. + _Py_UOpsAbstractFrame *frame; + _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; + int curr_frame_depth; + + // Arena for the symbolic types. + ty_arena t_arena; + + /* To do -- We could make this more space efficient + * by using a single array and growing the stack and + * locals toward each other. */ + _JitOptRefBuffer locals; + _JitOptRefBuffer stack; + JitOptRef locals_array[ABSTRACT_INTERP_LOCALS_SIZE]; + JitOptRef stack_array[ABSTRACT_INTERP_STACK_SIZE]; + _PyJitUopBuffer out_buffer; + _PyBloomFilter *dependencies; +} JitOptContext; + + +static inline void +uop_buffer_init(_PyJitUopBuffer *trace, _PyUOpInstruction *start, uint32_t size) +{ + trace->next = trace->start = start; + trace->end = start + size; +} + +static inline _PyUOpInstruction * +uop_buffer_last(_PyJitUopBuffer *trace) +{ + assert(trace->next > trace->start); + return trace->next-1; +} + +static inline int +uop_buffer_length(_PyJitUopBuffer *trace) +{ + return (int)(trace->next - trace->start); +} + +static inline int +uop_buffer_remaining_space(_PyJitUopBuffer *trace) +{ + return (int)(trace->end - trace->next); +} + +typedef struct _PyJitTracerInitialState { + int stack_depth; + int chain_depth; + struct _PyExitData *exit; + PyCodeObject *code; // Strong + PyFunctionObject *func; // Strong + struct _PyExecutorObject *executor; // Strong + _Py_CODEUNIT *start_instr; + _Py_CODEUNIT *close_loop_instr; + _Py_CODEUNIT *jump_backward_instr; +} _PyJitTracerInitialState; + +#define MAX_RECORDED_VALUES 3 +typedef struct _PyJitTracerPreviousState { + int instr_oparg; + int instr_stacklevel; + _Py_CODEUNIT *instr; + PyCodeObject *instr_code; // Strong + struct _PyInterpreterFrame *instr_frame; + PyObject *recorded_values[MAX_RECORDED_VALUES]; // Strong, may be NULL + int recorded_count; +} _PyJitTracerPreviousState; + +typedef struct _PyJitTracerTranslatorState { + int32_t fitness; // Current trace fitness, starts high, decrements + int frame_depth; // Current inline depth (0 = root frame) +} _PyJitTracerTranslatorState; + +typedef struct _PyJitTracerState { + bool is_tracing; + _PyJitTracerInitialState initial_state; + _PyJitTracerPreviousState prev_state; + _PyJitTracerTranslatorState translator_state; + JitOptContext opt_context; + _PyJitUopBuffer code_buffer; + _PyJitUopBuffer out_buffer; + _PyUOpInstruction uop_array[2 * UOP_MAX_TRACE_LENGTH]; +} _PyJitTracerState; typedef struct _PyExecutorLinkListNode { struct _PyExecutorObject *next; struct _PyExecutorObject *previous; } _PyExecutorLinkListNode; - -/* Bloom filter with m = 256 - * https://en.wikipedia.org/wiki/Bloom_filter */ -#define _Py_BLOOM_FILTER_WORDS 8 - -typedef struct { - uint32_t bits[_Py_BLOOM_FILTER_WORDS]; -} _PyBloomFilter; - typedef struct { uint8_t opcode; uint8_t oparg; - uint8_t valid:1; - uint8_t linked:1; - uint8_t chain_depth:6; // Must be big enough for MAX_CHAIN_DEPTH - 1. - bool warm; - int index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). - _PyBloomFilter bloom; - _PyExecutorLinkListNode links; + uint8_t valid; + uint8_t chain_depth; // Must be big enough for MAX_CHAIN_DEPTH - 1. + bool cold; + uint8_t pending_deletion; + int32_t index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). + int32_t bloom_array_idx; // Index in interp->executor_blooms/executor_ptrs. + _PyExecutorLinkListNode links; // Used by deletion list. PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR). } _PyVMData; -/* Depending on the format, - * the 32 bits between the oparg and operand are: - * UOP_FORMAT_TARGET: - * uint32_t target; - * UOP_FORMAT_JUMP - * uint16_t jump_target; - * uint16_t error_target; - */ -typedef struct { - uint16_t opcode:15; - uint16_t format:1; - uint16_t oparg; - union { - uint32_t target; - struct { - uint16_t jump_target; - uint16_t error_target; - }; - }; - uint64_t operand0; // A cache entry - uint64_t operand1; -#ifdef Py_STATS - uint64_t execution_count; -#endif -} _PyUOpInstruction; - -typedef struct { +typedef struct _PyExitData { uint32_t target; + uint16_t index:12; + uint16_t stack_cache:2; + uint16_t is_dynamic:1; + uint16_t is_control_flow:1; _Py_BackoffCounter temperature; struct _PyExecutorObject *executor; } _PyExitData; @@ -81,24 +198,96 @@ typedef struct _PyExecutorObject { uint32_t code_size; size_t jit_size; void *jit_code; - void *jit_side_entry; + _PyJitCodeRegistration *jit_registration; _PyExitData exits[1]; } _PyExecutorObject; -/* If pending deletion list gets large enough, then scan, - * and free any executors that aren't executing - * i.e. any that aren't a thread's current_executor. */ -#define EXECUTOR_DELETE_LIST_MAX 100 - // Export for '_opcode' shared extension (JIT compiler). PyAPI_FUNC(_PyExecutorObject*) _Py_GetExecutor(PyCodeObject *code, int offset); -void _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); +int _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); void _Py_ExecutorDetach(_PyExecutorObject *); -void _Py_BloomFilter_Init(_PyBloomFilter *); -void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); +/* We use a bloomfilter with k = 6, m = 256 + * The choice of k and the following constants + * could do with a more rigorous analysis, + * but here is a simple analysis: + * + * We want to keep the false positive rate low. + * For n = 5 (a trace depends on 5 objects), + * we expect 30 bits set, giving a false positive + * rate of (30/256)**6 == 2.5e-6 which is plenty + * good enough. + * + * However with n = 10 we expect 60 bits set (worst case), + * giving a false positive of (60/256)**6 == 0.0001 + * + * We choose k = 6, rather than a higher number as + * it means the false positive rate grows slower for high n. + * + * n = 5, k = 6 => fp = 2.6e-6 + * n = 5, k = 8 => fp = 3.5e-7 + * n = 10, k = 6 => fp = 1.6e-4 + * n = 10, k = 8 => fp = 0.9e-4 + * n = 15, k = 6 => fp = 0.18% + * n = 15, k = 8 => fp = 0.23% + * n = 20, k = 6 => fp = 1.1% + * n = 20, k = 8 => fp = 2.3% + * + * The above analysis assumes perfect hash functions, + * but those don't exist, so the real false positive + * rates may be worse. + */ + +#define _Py_BLOOM_FILTER_K 6 +#define _Py_BLOOM_FILTER_SEED 20221211 + +static inline uint64_t +address_to_hash(void *ptr) { + assert(ptr != NULL); + uint64_t uhash = _Py_BLOOM_FILTER_SEED; + uintptr_t addr = (uintptr_t)ptr; + for (int i = 0; i < SIZEOF_VOID_P; i++) { + uhash ^= addr & 255; + uhash *= (uint64_t)PyHASH_MULTIPLIER; + addr >>= 8; + } + return uhash; +} + +static inline void +_Py_BloomFilter_Init(_PyBloomFilter *bloom) +{ + for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { + bloom->bits[i] = 0; + } +} + +static inline void +_Py_BloomFilter_Add(_PyBloomFilter *bloom, void *ptr) +{ + uint64_t hash = address_to_hash(ptr); + assert(_Py_BLOOM_FILTER_K <= 8); + for (int i = 0; i < _Py_BLOOM_FILTER_K; i++) { + uint8_t bits = hash & 255; + bloom->bits[bits >> _Py_BLOOM_FILTER_WORD_SHIFT] |= + (_Py_bloom_filter_word_t)1 << (bits & (_Py_BLOOM_FILTER_BITS_PER_WORD - 1)); + hash >>= 8; + } +} + +static inline bool +bloom_filter_may_contain(const _PyBloomFilter *bloom, const _PyBloomFilter *hashes) +{ + for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { + if ((bloom->bits[i] & hashes->bits[i]) != hashes->bits[i]) { + return false; + } + } + return true; +} + #define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 #define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6 @@ -110,24 +299,20 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); #else # define _Py_Executors_InvalidateDependency(A, B, C) ((void)0) # define _Py_Executors_InvalidateAll(A, B) ((void)0) -# define _Py_Executors_InvalidateCold(A) ((void)0) #endif // Used as the threshold to trigger executor invalidation when -// trace_run_counter is greater than this value. -#define JIT_CLEANUP_THRESHOLD 100000 - -// This is the length of the trace we project initially. -#define UOP_MAX_TRACE_LENGTH 800 - -#define TRACE_STACK_SIZE 5 +// executor_creation_counter is greater than this value. +// This value is arbitrary and was not optimized. +#define JIT_CLEANUP_THRESHOLD 1000 -int _Py_uop_analyze_and_optimize(_PyInterpreterFrame *frame, - _PyUOpInstruction *trace, int trace_len, int curr_stackentries, - _PyBloomFilter *dependencies); +int _Py_uop_analyze_and_optimize( + _PyThreadStateImpl *tstate, + _PyUOpInstruction *input, int trace_len, int curr_stackentries, + _PyUOpInstruction *output, _PyBloomFilter *dependencies); -extern PyTypeObject _PyUOpExecutor_Type; +PyAPI_DATA(PyTypeObject) _PyUOpExecutor_Type; #define UOP_FORMAT_TARGET 0 @@ -151,90 +336,16 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) return inst->error_target; } -// Holds locals, stack, locals, stack ... co_consts (in that order) -#define MAX_ABSTRACT_INTERP_SIZE 4096 - -#define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * 5) - -// Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) -#define MAX_ABSTRACT_FRAME_DEPTH (TRACE_STACK_SIZE + 2) - -// The maximum number of side exits that we can take before requiring forward -// progress (and inserting a new ENTER_EXECUTOR instruction). In practice, this -// is the "maximum amount of polymorphism" that an isolated trace tree can -// handle before rejoining the rest of the program. -#define MAX_CHAIN_DEPTH 4 - -/* Symbols */ -/* See explanation in optimizer_symbols.c */ - - -typedef enum _JitSymType { - JIT_SYM_UNKNOWN_TAG = 1, - JIT_SYM_NULL_TAG = 2, - JIT_SYM_NON_NULL_TAG = 3, - JIT_SYM_BOTTOM_TAG = 4, - JIT_SYM_TYPE_VERSION_TAG = 5, - JIT_SYM_KNOWN_CLASS_TAG = 6, - JIT_SYM_KNOWN_VALUE_TAG = 7, - JIT_SYM_TUPLE_TAG = 8, - JIT_SYM_TRUTHINESS_TAG = 9, - JIT_SYM_COMPACT_INT = 10, -} JitSymType; - -typedef struct _jit_opt_known_class { - uint8_t tag; - uint32_t version; - PyTypeObject *type; -} JitOptKnownClass; - -typedef struct _jit_opt_known_version { - uint8_t tag; - uint32_t version; -} JitOptKnownVersion; - -typedef struct _jit_opt_known_value { - uint8_t tag; - PyObject *value; -} JitOptKnownValue; - -#define MAX_SYMBOLIC_TUPLE_SIZE 7 - -typedef struct _jit_opt_tuple { - uint8_t tag; - uint8_t length; - uint16_t items[MAX_SYMBOLIC_TUPLE_SIZE]; -} JitOptTuple; - -typedef struct { - uint8_t tag; - bool invert; - uint16_t value; -} JitOptTruthiness; - -typedef struct { - uint8_t tag; -} JitOptCompactInt; - -typedef union _jit_opt_symbol { - uint8_t tag; - JitOptKnownClass cls; - JitOptKnownValue value; - JitOptKnownVersion version; - JitOptTuple tuple; - JitOptTruthiness truthiness; - JitOptCompactInt compact; -} JitOptSymbol; - - -// This mimics the _PyStackRef API -typedef union { - uintptr_t bits; -} JitOptRef; #define REF_IS_BORROWED 1 +#define REF_IS_UNIQUE 2 +#define REF_IS_INVALID 3 +#define REF_TAG_BITS 3 + +#define REF_GET_TAG(x) ((uintptr_t)(x) & (REF_TAG_BITS)) +#define REF_CLEAR_TAG(x) ((uintptr_t)(x) & (~REF_TAG_BITS)) -#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~REF_IS_BORROWED))) +#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~REF_TAG_BITS))) static inline JitOptSymbol * PyJitRef_Unwrap(JitOptRef ref) @@ -252,59 +363,72 @@ PyJitRef_Wrap(JitOptSymbol *sym) } static inline JitOptRef -PyJitRef_Borrow(JitOptRef ref) +PyJitRef_WrapInvalid(void *ptr) { - return (JitOptRef){ .bits = ref.bits | REF_IS_BORROWED }; + return (JitOptRef){.bits = REF_CLEAR_TAG((uintptr_t)ptr) | REF_IS_INVALID}; } -static const JitOptRef PyJitRef_NULL = {.bits = REF_IS_BORROWED}; - static inline bool -PyJitRef_IsNull(JitOptRef ref) +PyJitRef_IsInvalid(JitOptRef ref) { - return ref.bits == PyJitRef_NULL.bits; + return REF_GET_TAG(ref.bits) == REF_IS_INVALID; } -static inline int -PyJitRef_IsBorrowed(JitOptRef ref) +static inline JitOptRef +PyJitRef_MakeUnique(JitOptRef ref) { - return (ref.bits & REF_IS_BORROWED) == REF_IS_BORROWED; + return (JitOptRef){ REF_CLEAR_TAG(ref.bits) | REF_IS_UNIQUE }; } -struct _Py_UOpsAbstractFrame { - // Max stacklen - int stack_len; - int locals_len; +static inline bool +PyJitRef_IsUnique(JitOptRef ref) +{ + return REF_GET_TAG(ref.bits) == REF_IS_UNIQUE; +} - JitOptRef *stack_pointer; - JitOptRef *stack; - JitOptRef *locals; -}; +static inline JitOptRef +PyJitRef_StripBorrowInfo(JitOptRef ref) +{ + if (PyJitRef_IsUnique(ref)) { + return ref; + } + return (JitOptRef){ .bits = REF_CLEAR_TAG(ref.bits) }; +} -typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; +static inline JitOptRef +PyJitRef_StripReferenceInfo(JitOptRef ref) +{ + return PyJitRef_Wrap(PyJitRef_Unwrap(ref)); +} -typedef struct ty_arena { - int ty_curr_number; - int ty_max_number; - JitOptSymbol arena[TY_ARENA_SIZE]; -} ty_arena; +static inline JitOptRef +PyJitRef_RemoveUnique(JitOptRef ref) +{ + if (PyJitRef_IsUnique(ref)) { + ref = PyJitRef_StripReferenceInfo(ref); + } + return ref; +} -typedef struct _JitOptContext { - char done; - char out_of_space; - bool contradiction; - // The current "executing" frame. - _Py_UOpsAbstractFrame *frame; - _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; - int curr_frame_depth; +static inline JitOptRef +PyJitRef_Borrow(JitOptRef ref) +{ + return (JitOptRef){ .bits = REF_CLEAR_TAG(ref.bits) | REF_IS_BORROWED }; +} - // Arena for the symbolic types. - ty_arena t_arena; +static const JitOptRef PyJitRef_NULL = {.bits = REF_IS_BORROWED}; - JitOptRef *n_consumed; - JitOptRef *limit; - JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; -} JitOptContext; +static inline bool +PyJitRef_IsNull(JitOptRef ref) +{ + return ref.bits == PyJitRef_NULL.bits; +} + +static inline int +PyJitRef_IsBorrowed(JitOptRef ref) +{ + return REF_GET_TAG(ref.bits) == REF_IS_BORROWED; +} extern bool _Py_uop_sym_is_null(JitOptRef sym); extern bool _Py_uop_sym_is_not_null(JitOptRef sym); @@ -318,11 +442,13 @@ extern JitOptRef _Py_uop_sym_new_type( extern JitOptRef _Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val); extern JitOptRef _Py_uop_sym_new_const_steal(JitOptContext *ctx, PyObject *const_val); bool _Py_uop_sym_is_safe_const(JitOptContext *ctx, JitOptRef sym); +bool _Py_uop_sym_is_not_container(JitOptRef sym); _PyStackRef _Py_uop_sym_get_const_as_stackref(JitOptContext *ctx, JitOptRef sym); extern JitOptRef _Py_uop_sym_new_null(JitOptContext *ctx); extern bool _Py_uop_sym_has_type(JitOptRef sym); extern bool _Py_uop_sym_matches_type(JitOptRef sym, PyTypeObject *typ); extern bool _Py_uop_sym_matches_type_version(JitOptRef sym, unsigned int version); +extern unsigned int _Py_uop_sym_get_type_version(JitOptRef sym); extern void _Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef sym); extern void _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptRef sym); extern void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef sym, PyTypeObject *typ); @@ -332,40 +458,98 @@ extern bool _Py_uop_sym_is_bottom(JitOptRef sym); extern int _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptRef sym); extern PyTypeObject *_Py_uop_sym_get_type(JitOptRef sym); extern JitOptRef _Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptRef *args); -extern JitOptRef _Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef sym, int item); -extern int _Py_uop_sym_tuple_length(JitOptRef sym); +extern JitOptRef _Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef sym, Py_ssize_t item); +extern Py_ssize_t _Py_uop_sym_tuple_length(JitOptRef sym); extern JitOptRef _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef value, bool truthy); extern bool _Py_uop_sym_is_compact_int(JitOptRef sym); extern JitOptRef _Py_uop_sym_new_compact_int(JitOptContext *ctx); extern void _Py_uop_sym_set_compact_int(JitOptContext *ctx, JitOptRef sym); - -extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); +extern JitOptRef _Py_uop_sym_new_predicate(JitOptContext *ctx, JitOptRef lhs_ref, JitOptRef rhs_ref, JitOptPredicateKind kind); +extern void _Py_uop_sym_apply_predicate_narrowing(JitOptContext *ctx, JitOptRef sym, bool branch_is_true); +extern void _Py_uop_sym_set_recorded_value(JitOptContext *ctx, JitOptRef sym, PyObject *value); +extern void _Py_uop_sym_set_recorded_type(JitOptContext *ctx, JitOptRef sym, PyTypeObject *type); +extern void _Py_uop_sym_set_recorded_gen_func(JitOptContext *ctx, JitOptRef ref, PyFunctionObject *value); +extern PyCodeObject *_Py_uop_sym_get_probable_func_code(JitOptRef sym); +extern PyObject *_Py_uop_sym_get_probable_value(JitOptRef sym); +extern PyTypeObject *_Py_uop_sym_get_probable_type(JitOptRef sym); +extern JitOptRef *_Py_uop_sym_set_stack_depth(JitOptContext *ctx, int stack_depth, JitOptRef *current_sp); + +extern void _Py_uop_abstractcontext_init(JitOptContext *ctx, _PyBloomFilter *dependencies); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( JitOptContext *ctx, PyCodeObject *co, - int curr_stackentries, JitOptRef *args, int arg_len); -extern int _Py_uop_frame_pop(JitOptContext *ctx); + +extern _Py_UOpsAbstractFrame *_Py_uop_frame_new_from_symbol( + JitOptContext *ctx, + JitOptRef callable, + JitOptRef *args, + int arg_len); + +extern int _Py_uop_frame_pop(JitOptContext *ctx, PyCodeObject *co); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); -PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyExecutorObject **exec_ptr, int chain_depth); +PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, PyThreadState *tstate); -static inline int is_terminator(const _PyUOpInstruction *uop) +static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit) { - int opcode = uop->opcode; - return ( - opcode == _EXIT_TRACE || - opcode == _JUMP_TO_TOP - ); + _PyExitData *exit0 = exit - exit->index; + return (_PyExecutorObject *)(((char *)exit0) - offsetof(_PyExecutorObject, exits)); } +extern _PyExecutorObject *_PyExecutor_GetColdExecutor(void); +extern _PyExecutorObject *_PyExecutor_GetColdDynamicExecutor(void); + +PyAPI_FUNC(void) _PyExecutor_ClearExit(_PyExitData *exit); + +extern void _PyExecutor_Free(_PyExecutorObject *self); + PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); #ifdef _Py_TIER2 -extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); +PyAPI_FUNC(void) _Py_ClearExecutorDeletionList(PyInterpreterState *interp); +#endif + +PyAPI_FUNC(int) _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int stop_tracing_opcode); + +PyAPI_FUNC(int) +_PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *start_instr, + _Py_CODEUNIT *close_loop_instr, _PyStackRef *stack_pointer, int chain_depth, _PyExitData *exit, + int oparg, _PyExecutorObject *current_executor); + +PyAPI_FUNC(void) _PyJit_FinalizeTracing(PyThreadState *tstate, int err); +PyAPI_FUNC(bool) _PyJit_EnterExecutorShouldStopTracing(int og_opcode); + +void _PyPrintExecutor(_PyExecutorObject *executor, const _PyUOpInstruction *marker); +void _PyJit_TracerFree(_PyThreadStateImpl *_tstate); + +#ifdef _Py_TIER2 +typedef void (*_Py_RecordFuncPtr)(_PyInterpreterFrame *frame, _PyStackRef *stackpointer, int oparg, PyObject **recorded_value); +PyAPI_DATA(const _Py_RecordFuncPtr) _PyOpcode_RecordFunctions[]; + +typedef struct { + uint8_t count; + uint8_t indices[MAX_RECORDED_VALUES]; +} _PyOpcodeRecordEntry; + +typedef struct { + uint8_t count; + uint8_t transform_mask; + uint8_t slots[MAX_RECORDED_VALUES]; +} _PyOpcodeRecordSlotMap; + +PyAPI_DATA(const _PyOpcodeRecordEntry) _PyOpcode_RecordEntries[256]; +PyAPI_DATA(const _PyOpcodeRecordSlotMap) _PyOpcode_RecordSlotMaps[256]; + +/* Convert a family-recorded value to the form a recorder uop expects. + * If no transform is needed, return the input value unchanged. + * Takes ownership of `value` and returns a new strong reference or NULL. + */ +PyAPI_FUNC(PyObject *) _PyOpcode_RecordTransformValue(int uop, PyObject *value); #endif #ifdef __cplusplus diff --git a/Include/internal/pycore_optimizer_types.h b/Include/internal/pycore_optimizer_types.h new file mode 100644 index 000000000000000..a722652cc8163aa --- /dev/null +++ b/Include/internal/pycore_optimizer_types.h @@ -0,0 +1,169 @@ +#ifndef Py_INTERNAL_OPTIMIZER_TYPES_H +#define Py_INTERNAL_OPTIMIZER_TYPES_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include +#include "pycore_uop.h" // UOP_MAX_TRACE_LENGTH + +#define ABSTRACT_INTERP_STACK_SIZE 256 +#define ABSTRACT_INTERP_LOCALS_SIZE 512 + + +#define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * 5) + +// Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) +#define MAX_ABSTRACT_FRAME_DEPTH (16) + +// The maximum number of side exits that we can take before requiring forward +// progress (and inserting a new ENTER_EXECUTOR instruction). In practice, this +// is the "maximum amount of polymorphism" that an isolated trace tree can +// handle before rejoining the rest of the program. +#define MAX_CHAIN_DEPTH 4 + +/* Symbols */ +/* See explanation in optimizer_symbols.c */ + + +typedef enum _JitSymType { + JIT_SYM_UNKNOWN_TAG = 1, + JIT_SYM_NULL_TAG = 2, + JIT_SYM_NON_NULL_TAG = 3, + JIT_SYM_BOTTOM_TAG = 4, + JIT_SYM_TYPE_VERSION_TAG = 5, + JIT_SYM_KNOWN_CLASS_TAG = 7, + JIT_SYM_KNOWN_VALUE_TAG = 8, + JIT_SYM_TUPLE_TAG = 9, + JIT_SYM_TRUTHINESS_TAG = 10, + JIT_SYM_COMPACT_INT = 11, + JIT_SYM_PREDICATE_TAG = 12, + JIT_SYM_RECORDED_VALUE_TAG = 13, + JIT_SYM_RECORDED_TYPE_TAG = 14, + JIT_SYM_RECORDED_GEN_FUNC_TAG = 15, +} JitSymType; + +typedef struct _jit_opt_known_class { + uint8_t tag; + uint32_t version; + PyTypeObject *type; +} JitOptKnownClass; + +typedef struct _jit_opt_known_version { + uint8_t tag; + uint32_t version; +} JitOptKnownVersion; + +typedef struct _jit_opt_known_func_version { + uint8_t tag; + uint32_t func_version; +} JitOptKnownFuncVersion; + +typedef struct _jit_opt_known_value { + uint8_t tag; + PyObject *value; +} JitOptKnownValue; + +#define MAX_SYMBOLIC_TUPLE_SIZE 7 + +typedef struct _jit_opt_tuple { + uint8_t tag; + uint8_t length; + uint16_t items[MAX_SYMBOLIC_TUPLE_SIZE]; +} JitOptTuple; + +typedef struct { + uint8_t tag; + bool invert; + uint16_t value; +} JitOptTruthiness; + +typedef enum { + JIT_PRED_IS, + JIT_PRED_IS_NOT, + JIT_PRED_EQ, + JIT_PRED_NE, +} JitOptPredicateKind; + +typedef struct { + uint8_t tag; + uint8_t kind; + uint16_t lhs; + uint16_t rhs; +} JitOptPredicate; + +typedef struct _jit_opt_recorded_value { + uint8_t tag; + bool known_type; + PyObject *value; +} JitOptRecordedValue; + +typedef struct _jit_opt_recorded_type { + uint8_t tag; + PyTypeObject *type; +} JitOptRecordedType; + +/* Represents a generator, but we record the + * function as the generator is emphemeral */ +typedef struct _jit_opt_recorded_gen_func { + uint8_t tag; + PyFunctionObject *func; +} JitOptRecordedGenFunc; + +typedef struct { + uint8_t tag; +} JitOptCompactInt; + +typedef union _jit_opt_symbol { + uint8_t tag; + JitOptKnownClass cls; + JitOptKnownValue value; + JitOptKnownVersion version; + JitOptKnownFuncVersion func_version; + JitOptTuple tuple; + JitOptTruthiness truthiness; + JitOptCompactInt compact; + JitOptPredicate predicate; + JitOptRecordedValue recorded_value; + JitOptRecordedType recorded_type; + JitOptRecordedGenFunc recorded_gen_func; +} JitOptSymbol; + +// This mimics the _PyStackRef API +typedef union { + uintptr_t bits; +} JitOptRef; + +typedef struct _Py_UOpsAbstractFrame { + bool globals_watched; + // The version number of the globals dicts, once checked. 0 if unchecked. + uint32_t globals_checked_version; + // Max stacklen + int stack_len; + int locals_len; + bool caller; // We have made a call from this frame during the trace + bool is_c_recursion_checked; + JitOptRef callable; + PyFunctionObject *func; + PyCodeObject *code; + + JitOptRef *stack_pointer; + JitOptRef *stack; + JitOptRef *locals; +} _Py_UOpsAbstractFrame; + +typedef struct ty_arena { + int ty_curr_number; + int ty_max_number; + JitOptSymbol arena[TY_ARENA_SIZE]; +} ty_arena; + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_OPTIMIZER_TYPES_H */ diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index 2885dee63dcf94b..b89d02035db71f9 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -14,10 +14,8 @@ extern "C" { #include "pycore_pyarena.h" // PyArena _Py_DECLARE_STR(empty, "") -#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED) #define _parser_runtime_state_INIT \ { \ - .mutex = {0}, \ .dummy_name = { \ .kind = Name_kind, \ .v.Name.id = &_Py_STR(empty), \ @@ -28,27 +26,14 @@ _Py_DECLARE_STR(empty, "") .end_col_offset = 0, \ }, \ } -#else -#define _parser_runtime_state_INIT \ - { \ - .dummy_name = { \ - .kind = Name_kind, \ - .v.Name.id = &_Py_STR(empty), \ - .v.Name.ctx = Load, \ - .lineno = 1, \ - .col_offset = 0, \ - .end_lineno = 1, \ - .end_col_offset = 0, \ - }, \ - } -#endif extern struct _mod* _PyParser_ASTFromString( const char *str, PyObject* filename, int mode, PyCompilerFlags *flags, - PyArena *arena); + PyArena *arena, + PyObject *module); extern struct _mod* _PyParser_ASTFromFile( FILE *fp, diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 3e41e2fd1569caf..d8ec306a0dae3fc 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -31,6 +31,8 @@ extern "C" { _Py_atomic_store_ptr(&value, new_value) #define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) \ _Py_atomic_load_ptr_acquire(&value) +#define FT_ATOMIC_LOAD_PTR_CONSUME(value) \ + _Py_atomic_load_ptr_consume(&value) #define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) \ _Py_atomic_load_uintptr_acquire(&value) #define FT_ATOMIC_LOAD_PTR_RELAXED(value) \ @@ -39,12 +41,18 @@ extern "C" { _Py_atomic_load_uint8(&value) #define FT_ATOMIC_STORE_UINT8(value, new_value) \ _Py_atomic_store_uint8(&value, new_value) +#define FT_ATOMIC_LOAD_INT8_RELAXED(value) \ + _Py_atomic_load_int8_relaxed(&value) #define FT_ATOMIC_LOAD_UINT8_RELAXED(value) \ _Py_atomic_load_uint8_relaxed(&value) #define FT_ATOMIC_LOAD_UINT16_RELAXED(value) \ _Py_atomic_load_uint16_relaxed(&value) #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) \ _Py_atomic_load_uint32_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT64_ACQUIRE(value) \ + _Py_atomic_load_uint64_acquire(&value) +#define FT_ATOMIC_LOAD_UINT64_RELAXED(value) \ + _Py_atomic_load_uint64_relaxed(&value) #define FT_ATOMIC_LOAD_ULONG_RELAXED(value) \ _Py_atomic_load_ulong_relaxed(&value) #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ @@ -53,14 +61,26 @@ extern "C" { _Py_atomic_store_ptr_release(&value, new_value) #define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) \ _Py_atomic_store_uintptr_release(&value, new_value) +#define FT_ATOMIC_STORE_INT8_RELAXED(value, new_value) \ + _Py_atomic_store_int8_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_INT8_RELEASE(value, new_value) \ + _Py_atomic_store_int8_release(&value, new_value) #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \ _Py_atomic_store_ssize_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) \ + _Py_atomic_store_ssize_release(&value, new_value) #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) \ _Py_atomic_store_uint8_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) \ _Py_atomic_store_uint16_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \ _Py_atomic_store_uint32_relaxed(&value, new_value) +#define FT_ATOMIC_AND_UINT64(value, new_value) \ + (void)_Py_atomic_and_uint64(&value, new_value) +#define FT_ATOMIC_OR_UINT64(value, new_value) \ + (void)_Py_atomic_or_uint64(&value, new_value) +#define FT_ATOMIC_ADD_UINT64(value, new_value) \ + (void)_Py_atomic_add_uint64(&value, new_value) #define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) \ _Py_atomic_store_char_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \ @@ -77,10 +97,16 @@ extern "C" { _Py_atomic_store_ushort_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_USHORT_RELAXED(value) \ _Py_atomic_load_ushort_relaxed(&value) +#define FT_ATOMIC_LOAD_INT(value) \ + _Py_atomic_load_int(&value) +#define FT_ATOMIC_STORE_INT(value, new_value) \ + _Py_atomic_store_int(&value, new_value) #define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) \ _Py_atomic_store_int_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_INT_RELAXED(value) \ _Py_atomic_load_int_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT(value) \ + _Py_atomic_load_uint(&value) #define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) \ _Py_atomic_store_uint_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_UINT_RELAXED(value) \ @@ -111,6 +137,9 @@ extern "C" { _Py_atomic_load_ullong_relaxed(&value) #define FT_ATOMIC_ADD_SSIZE(value, new_value) \ (void)_Py_atomic_add_ssize(&value, new_value) +#define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) +#define FT_MUTEX_LOCK_FLAGS(lock, flags) PyMutex_LockFlags(lock, flags) +#define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -119,21 +148,31 @@ extern "C" { #define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) value #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value #define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_PTR_CONSUME(value) value #define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) value #define FT_ATOMIC_LOAD_PTR_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT8(value) value #define FT_ATOMIC_STORE_UINT8(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_INT8_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT16_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT64_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_UINT64_RELAXED(value) value #define FT_ATOMIC_LOAD_ULONG_RELAXED(value) value #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_INT8_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_INT8_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_AND_UINT64(value, new_value) (void)(value &= new_value) +#define FT_ATOMIC_OR_UINT64(value, new_value) (void)(value |= new_value) +#define FT_ATOMIC_ADD_UINT64(value, new_value) (void)(value += new_value) #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) value #define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) value @@ -142,8 +181,11 @@ extern "C" { #define FT_ATOMIC_STORE_SHORT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_USHORT_RELAXED(value) value #define FT_ATOMIC_STORE_USHORT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_INT(value) value +#define FT_ATOMIC_STORE_INT(value, new_value) value = new_value #define FT_ATOMIC_LOAD_INT_RELAXED(value) value #define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UINT(value) value #define FT_ATOMIC_LOAD_UINT_RELAXED(value) value #define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_LONG_RELAXED(value) value @@ -159,6 +201,9 @@ extern "C" { #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value #define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_ADD_SSIZE(value, new_value) (void)(value += new_value) +#define FT_MUTEX_LOCK(lock) do {} while (0) +#define FT_MUTEX_LOCK_FLAGS(lock, flags) do {} while (0) +#define FT_MUTEX_UNLOCK(lock) do {} while (0) #endif diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 2c2048f7e1272a3..e436aa6bf12cb27 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -11,8 +11,8 @@ extern "C" { /* Error handling definitions */ -extern _PyErr_StackItem* _PyErr_GetTopmostException(PyThreadState *tstate); -extern PyObject* _PyErr_GetHandledException(PyThreadState *); +PyAPI_FUNC(_PyErr_StackItem*) _PyErr_GetTopmostException(PyThreadState *tstate); +PyAPI_FUNC(PyObject*) _PyErr_GetHandledException(PyThreadState *); extern void _PyErr_SetHandledException(PyThreadState *, PyObject *); extern void _PyErr_GetExcInfo(PyThreadState *, PyObject **, PyObject **, PyObject **); @@ -29,7 +29,8 @@ PyAPI_FUNC(PyObject*) _PyErr_FormatFromCause( ... ); -extern int _PyException_AddNote( +// Export for 'pyexpat' shared extension. +PyAPI_FUNC(int) _PyException_AddNote( PyObject *exc, PyObject *note); @@ -108,7 +109,7 @@ extern void _PyErr_Restore( PyObject *value, PyObject *traceback); -extern void _PyErr_SetObject( +PyAPI_FUNC(void) _PyErr_SetObject( PyThreadState *tstate, PyObject *type, PyObject *value); @@ -123,7 +124,8 @@ extern void _PyErr_SetNone(PyThreadState *tstate, PyObject *exception); extern PyObject* _PyErr_NoMemory(PyThreadState *tstate); extern int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, - int end_lineno, int end_col_offset); + int end_lineno, int end_col_offset, + PyObject *module); extern void _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset); @@ -168,7 +170,8 @@ extern PyObject* _PyErr_FormatFromCauseTstate( const char *format, ...); -extern PyObject* _PyExc_CreateExceptionGroup( +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyExc_CreateExceptionGroup( const char *msg, PyObject *excs); @@ -179,7 +182,8 @@ extern PyObject* _PyExc_PrepReraiseStar( extern int _PyErr_CheckSignalsTstate(PyThreadState *tstate); extern void _Py_DumpExtensionModules(int fd, PyInterpreterState *interp); -extern PyObject* _Py_CalculateSuggestions(PyObject *dir, PyObject *name); +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _Py_CalculateSuggestions(PyObject *dir, PyObject *name); extern PyObject* _Py_Offer_Suggestions(PyObject* exception); // Export for '_testinternalcapi' shared extension diff --git a/Include/internal/pycore_pyhash.h b/Include/internal/pycore_pyhash.h index 84cb72fa6fd1b26..3056dc44cc0f1b1 100644 --- a/Include/internal/pycore_pyhash.h +++ b/Include/internal/pycore_pyhash.h @@ -27,14 +27,14 @@ _Py_HashPointerRaw(const void *ptr) * pppppppp ssssssss ........ fnv -- two Py_hash_t * k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t * ........ ........ ssssssss djbx33a -- 16 bytes padding + one Py_hash_t - * ........ ........ eeeeeeee pyexpat XML hash salt + * eeeeeeee eeeeeeee eeeeeeee pyexpat XML hash salt * * memory layout on 32 bit systems * cccccccc cccccccc cccccccc uc * ppppssss ........ ........ fnv -- two Py_hash_t * k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t (*) * ........ ........ ssss.... djbx33a -- 16 bytes padding + one Py_hash_t - * ........ ........ eeee.... pyexpat XML hash salt + * eeeeeeee eeeeeeee eeee.... pyexpat XML hash salt * * (*) The siphash member may not be available on 32 bit platforms without * an unsigned int64 data type. @@ -58,7 +58,9 @@ typedef union { Py_hash_t suffix; } djbx33a; struct { - unsigned char padding[16]; + /* 16 bytes for XML_SetHashSalt16Bytes */ + uint8_t hashsalt16[16]; + /* 4/8 bytes for legacy XML_SetHashSalt */ Py_hash_t hashsalt; } expat; } _Py_HashSecret_t; diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 8faf7a4d403f846..12e1e78526db35a 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -18,9 +18,6 @@ extern int _Py_SetFileSystemEncoding( const char *errors); extern void _Py_ClearFileSystemEncoding(void); extern PyStatus _PyUnicode_InitEncodings(PyThreadState *tstate); -#ifdef MS_WINDOWS -extern int _PyUnicode_EnableLegacyWindowsFSEncoding(void); -#endif extern int _Py_IsLocaleCoercionTarget(const char *ctype_loc); diff --git a/Include/internal/pycore_pymath.h b/Include/internal/pycore_pymath.h index eea8996ba68ca08..532c5ceafb56395 100644 --- a/Include/internal/pycore_pymath.h +++ b/Include/internal/pycore_pymath.h @@ -33,7 +33,7 @@ extern "C" { static inline void _Py_ADJUST_ERANGE1(double x) { if (errno == 0) { - if (x == Py_INFINITY || x == -Py_INFINITY) { + if (x == INFINITY || x == -INFINITY) { errno = ERANGE; } } @@ -44,8 +44,8 @@ static inline void _Py_ADJUST_ERANGE1(double x) static inline void _Py_ADJUST_ERANGE2(double x, double y) { - if (x == Py_INFINITY || x == -Py_INFINITY || - y == Py_INFINITY || y == -Py_INFINITY) + if (x == INFINITY || x == -INFINITY || + y == INFINITY || y == -INFINITY) { if (errno == 0) { errno = ERANGE; @@ -146,17 +146,17 @@ extern void _Py_set_387controlword(unsigned short); unsigned int old_fpcr, new_fpcr #define _Py_SET_53BIT_PRECISION_START \ do { \ - __asm__ ("fmove.l %%fpcr,%0" : "=g" (old_fpcr)); \ + __asm__ ("fmove.l %%fpcr,%0" : "=dm" (old_fpcr)); \ /* Set double precision / round to nearest. */ \ new_fpcr = (old_fpcr & ~0xf0) | 0x80; \ if (new_fpcr != old_fpcr) { \ - __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (new_fpcr));\ + __asm__ volatile ("fmove.l %0,%%fpcr" : : "dm" (new_fpcr)); \ } \ } while (0) #define _Py_SET_53BIT_PRECISION_END \ do { \ if (new_fpcr != old_fpcr) { \ - __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (old_fpcr)); \ + __asm__ volatile ("fmove.l %0,%%fpcr" : : "dm" (old_fpcr)); \ } \ } while (0) #endif @@ -182,8 +182,7 @@ extern void _Py_set_387controlword(unsigned short); // (extended precision), and we don't know how to change // the rounding precision. #if !defined(DOUBLE_IS_LITTLE_ENDIAN_IEEE754) && \ - !defined(DOUBLE_IS_BIG_ENDIAN_IEEE754) && \ - !defined(DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754) + !defined(DOUBLE_IS_BIG_ENDIAN_IEEE754) # define _PY_SHORT_FLOAT_REPR 0 #endif diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index f3f2ae0a140828b..05484e847f1195a 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -54,20 +54,43 @@ static inline int _PyMem_IsPtrFreed(const void *ptr) { uintptr_t value = (uintptr_t)ptr; #if SIZEOF_VOID_P == 8 - return (value == 0 + return (value <= 0xff // NULL, 0x1, 0x2, ..., 0xff || value == (uintptr_t)0xCDCDCDCDCDCDCDCD || value == (uintptr_t)0xDDDDDDDDDDDDDDDD - || value == (uintptr_t)0xFDFDFDFDFDFDFDFD); + || value == (uintptr_t)0xFDFDFDFDFDFDFDFD + || value >= (uintptr_t)0xFFFFFFFFFFFFFF00); // -0xff, ..., -2, -1 #elif SIZEOF_VOID_P == 4 - return (value == 0 + return (value <= 0xff || value == (uintptr_t)0xCDCDCDCD || value == (uintptr_t)0xDDDDDDDD - || value == (uintptr_t)0xFDFDFDFD); + || value == (uintptr_t)0xFDFDFDFD + || value >= (uintptr_t)0xFFFFFF00); #else # error "unknown pointer size" #endif } +// Similar to _PyMem_IsPtrFreed() but expects an 'unsigned long' instead of a +// pointer. +static inline int _PyMem_IsULongFreed(unsigned long value) +{ +#if SIZEOF_LONG == 8 + return (value == 0 + || value == (unsigned long)0xCDCDCDCDCDCDCDCD + || value == (unsigned long)0xDDDDDDDDDDDDDDDD + || value == (unsigned long)0xFDFDFDFDFDFDFDFD + || value == (unsigned long)0xFFFFFFFFFFFFFFFF); +#elif SIZEOF_LONG == 4 + return (value == 0 + || value == (unsigned long)0xCDCDCDCD + || value == (unsigned long)0xDDDDDDDD + || value == (unsigned long)0xFDFDFDFD + || value == (unsigned long)0xFFFFFFFF); +#else +# error "unknown long size" +#endif +} + extern int _PyMem_GetAllocatorName( const char *name, PyMemAllocatorName *allocator); diff --git a/Include/internal/pycore_pymem_init.h b/Include/internal/pycore_pymem_init.h index c593edc86d9952a..2a0e0817dcc7f86 100644 --- a/Include/internal/pycore_pymem_init.h +++ b/Include/internal/pycore_pymem_init.h @@ -30,6 +30,12 @@ extern void* _PyMem_MiCalloc(void *, size_t, size_t); extern void _PyMem_MiFree(void *, void *); extern void* _PyMem_MiRealloc(void *, void *, size_t); # define PYMEM_ALLOC {NULL, _PyMem_MiMalloc, _PyMem_MiCalloc, _PyMem_MiRealloc, _PyMem_MiFree} +extern void* _PyMem_MiRawMalloc(void *, size_t); +extern void* _PyMem_MiRawCalloc(void *, size_t, size_t); +extern void _PyMem_MiRawFree(void *, void *); +extern void* _PyMem_MiRawRealloc(void *, void *, size_t); +# undef PYRAW_ALLOC +# define PYRAW_ALLOC {NULL, _PyMem_MiRawMalloc, _PyMem_MiRawCalloc, _PyMem_MiRawRealloc, _PyMem_MiRawFree} #elif defined(WITH_PYMALLOC) extern void* _PyObject_Malloc(void *, size_t); extern void* _PyObject_Calloc(void *, size_t, size_t); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index ea3dfbd2eef9c15..ca6819d2cd44730 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -89,8 +89,9 @@ _Py_ThreadCanHandleSignals(PyInterpreterState *interp) /* Variable and static inline functions for in-line access to current thread and interpreter state */ -#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +#if !defined(Py_BUILD_CORE_MODULE) extern _Py_thread_local PyThreadState *_Py_tss_tstate; +extern _Py_thread_local PyInterpreterState *_Py_tss_interp; #endif #ifndef NDEBUG @@ -114,7 +115,7 @@ PyAPI_FUNC(PyThreadState *) _PyThreadState_GetCurrent(void); static inline PyThreadState* _PyThreadState_GET(void) { -#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +#if !defined(Py_BUILD_CORE_MODULE) return _Py_tss_tstate; #else return _PyThreadState_GetCurrent(); @@ -204,11 +205,15 @@ _Py_EnsureFuncTstateNotNULL(const char *func, PyThreadState *tstate) See also PyInterpreterState_Get() and _PyGILState_GetInterpreterStateUnsafe(). */ static inline PyInterpreterState* _PyInterpreterState_GET(void) { - PyThreadState *tstate = _PyThreadState_GET(); #ifdef Py_DEBUG + PyThreadState *tstate = _PyThreadState_GET(); _Py_EnsureTstateNotNULL(tstate); #endif - return tstate->interp; +#if !defined(Py_BUILD_CORE_MODULE) + return _Py_tss_interp; +#else + return _PyThreadState_GET()->interp; +#endif } @@ -261,6 +266,8 @@ extern int _PyOS_InterruptOccurred(PyThreadState *tstate); PyMutex_LockFlags(&(runtime)->interpreters.mutex, _Py_LOCK_DONT_DETACH) #define HEAD_UNLOCK(runtime) \ PyMutex_Unlock(&(runtime)->interpreters.mutex) +#define ASSERT_HEAD_IS_LOCKED(runtime) \ + assert(PyMutex_IsLocked(&(runtime)->interpreters.mutex)) #define _Py_FOR_EACH_TSTATE_UNLOCKED(interp, t) \ for (PyThreadState *t = interp->threads.head; t; t = t->next) @@ -309,15 +316,20 @@ static uintptr_t return_pointer_as_int(char* p) { static inline uintptr_t _Py_get_machine_stack_pointer(void) { -#if _Py__has_builtin(__builtin_frame_address) || defined(__GNUC__) - return (uintptr_t)__builtin_frame_address(0); -#elif defined(_MSC_VER) - return (uintptr_t)_AddressOfReturnAddress(); + uintptr_t result; +#if defined(_M_ARM64) + result = __getReg(31); +#elif defined(_M_X64) || defined(_M_IX86) + result = (uintptr_t)_AddressOfReturnAddress(); +#elif defined(__aarch64__) + __asm__ ("mov %0, sp" : "=r" (result)); +#elif defined(__x86_64__) + __asm__("{movq %%rsp, %0" : "=r" (result)); #else char here; - /* Avoid compiler warning about returning stack address */ - return return_pointer_as_int(&here); + result = (uintptr_t)&here; #endif + return result; } static inline intptr_t @@ -326,9 +338,27 @@ _Py_RecursionLimit_GetMargin(PyThreadState *tstate) _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; assert(_tstate->c_stack_hard_limit != 0); intptr_t here_addr = _Py_get_machine_stack_pointer(); +#if _Py_STACK_GROWS_DOWN return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, here_addr - (intptr_t)_tstate->c_stack_soft_limit, _PyOS_STACK_MARGIN_SHIFT); +#else + return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (intptr_t)_tstate->c_stack_soft_limit - here_addr, _PyOS_STACK_MARGIN_SHIFT); +#endif } +/* PEP 788 structures. */ + +struct PyInterpreterGuard { + PyInterpreterState *interp; +}; + +struct PyInterpreterView { + int64_t id; +}; + +// Exports for '_testinternalcapi' shared extension +PyAPI_FUNC(Py_ssize_t) _PyInterpreterState_GuardCountdown(PyInterpreterState *interp); +PyAPI_FUNC(PyInterpreterState *) _PyInterpreterGuard_GetInterpreter(PyInterpreterGuard *guard); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pystats.h b/Include/internal/pycore_pystats.h index f8af398a5605861..50ab21aa0f1902c 100644 --- a/Include/internal/pycore_pystats.h +++ b/Include/internal/pycore_pystats.h @@ -9,7 +9,7 @@ extern "C" { #endif #ifdef Py_STATS -extern void _Py_StatsOn(void); +extern int _Py_StatsOn(void); extern void _Py_StatsOff(void); extern void _Py_StatsClear(void); extern int _Py_PrintSpecializationStats(int to_file); diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index c2832098ddb3e7b..66dd7cd843b04fc 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -33,14 +33,21 @@ extern const char* _Py_SourceAsString( PyCompilerFlags *cf, PyObject **cmd_copy); +extern PyObject * _Py_CompileStringObjectWithModule( + const char *str, + PyObject *filename, int start, + PyCompilerFlags *flags, int optimize, + PyObject *module); + /* Stack size, in "pointers". This must be large enough, so * no two calls to check recursion depth are more than this far * apart. In practice, that means it must be larger than the C * stack consumption of PyEval_EvalDefault */ -#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) -# define _PyOS_LOG2_STACK_MARGIN 12 -#elif defined(Py_DEBUG) && defined(WIN32) +#if (defined(Py_DEBUG) \ + || defined(_Py_ADDRESS_SANITIZER) \ + || defined(_Py_THREAD_SANITIZER)) \ + || defined(_Py_UNDEFINED_BEHAVIOR_SANITIZER) # define _PyOS_LOG2_STACK_MARGIN 12 #else # define _PyOS_LOG2_STACK_MARGIN 11 @@ -54,6 +61,12 @@ extern const char* _Py_SourceAsString( # define _PyOS_STACK_MARGIN_SHIFT (_PyOS_LOG2_STACK_MARGIN + 2) #endif +#ifdef _Py_THREAD_SANITIZER +# define _PyOS_MIN_STACK_SIZE (_PyOS_STACK_MARGIN_BYTES * 6) +#else +# define _PyOS_MIN_STACK_SIZE (_PyOS_STACK_MARGIN_BYTES * 3) +#endif + #ifdef __cplusplus } diff --git a/Include/internal/pycore_qsbr.h b/Include/internal/pycore_qsbr.h index 1f9b3fcf777493f..eeca6fc472be37b 100644 --- a/Include/internal/pycore_qsbr.h +++ b/Include/internal/pycore_qsbr.h @@ -83,8 +83,9 @@ struct _qsbr_shared { // Minimum observed read sequence of all QSBR thread states uint64_t rd_seq; - // Array of QSBR thread states. + // Array of QSBR thread states (aligned to 64 bytes). struct _qsbr_pad *array; + void *array_raw; // raw allocation pointer (for free) Py_ssize_t size; // Freelist of unused _qsbr_thread_states (protected by mutex) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 7fc7f343fe600f8..fcd2ae9b1d1f1ae 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -56,6 +56,29 @@ _PyRuntimeState_SetFinalizing(_PyRuntimeState *runtime, PyThreadState *tstate) { } } +// Atomic so a thread that reads initialized=1 observes all writes +// from the initialization sequence (gh-146302). + +static inline int +_PyRuntimeState_GetCoreInitialized(_PyRuntimeState *runtime) { + return _Py_atomic_load_int(&runtime->core_initialized); +} + +static inline void +_PyRuntimeState_SetCoreInitialized(_PyRuntimeState *runtime, int initialized) { + _Py_atomic_store_int(&runtime->core_initialized, initialized); +} + +static inline int +_PyRuntimeState_GetInitialized(_PyRuntimeState *runtime) { + return _Py_atomic_load_int(&runtime->initialized); +} + +static inline void +_PyRuntimeState_SetInitialized(_PyRuntimeState *runtime, int initialized) { + _Py_atomic_store_int(&runtime->initialized, initialized); +} + #ifdef __cplusplus } diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index b182f7825a2326f..6c48ac0dccfcb16 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -13,7 +13,7 @@ extern "C" { #include "pycore_debug_offsets.h" // _Py_DebugOffsets_INIT() #include "pycore_dtoa.h" // _dtoa_state_INIT() #include "pycore_faulthandler.h" // _faulthandler_runtime_state_INIT -#include "pycore_floatobject.h" // _py_float_format_unknown +#include "pycore_floatobject.h" // _py_float_format_* #include "pycore_function.h" #include "pycore_hamt.h" // _PyHamt_BitmapNode_Type #include "pycore_import.h" // IMPORTS_INIT @@ -84,10 +84,6 @@ extern PyTypeObject _PyExc_MemoryError; .stoptheworld = { \ .is_global = 1, \ }, \ - .float_state = { \ - .float_format = _py_float_format_unknown, \ - .double_format = _py_float_format_unknown, \ - }, \ .types = { \ .next_version_tag = _Py_TYPE_VERSION_NEXT, \ }, \ @@ -134,13 +130,7 @@ extern PyTypeObject _PyExc_MemoryError; }, \ .gc = { \ .enabled = 1, \ - .young = { .threshold = 2000, }, \ - .old = { \ - { .threshold = 10, }, \ - { .threshold = 0, }, \ - }, \ - .work_to_do = -5000, \ - .phase = GC_PHASE_MARK, \ + GC_GENERATION_INIT \ }, \ .qsbr = { \ .wr_seq = QSBR_INITIAL, \ @@ -233,4 +223,4 @@ extern PyTypeObject _PyExc_MemoryError; #ifdef __cplusplus } #endif -#endif /* !Py_INTERNAL_RUNTIME_INIT_H */ \ No newline at end of file +#endif /* !Py_INTERNAL_RUNTIME_INIT_H */ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 85ced09d29d3387..75273243ef6df06 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -275,6 +275,774 @@ extern "C" { _PyLong_DIGIT_INIT(254), \ _PyLong_DIGIT_INIT(255), \ _PyLong_DIGIT_INIT(256), \ + _PyLong_DIGIT_INIT(257), \ + _PyLong_DIGIT_INIT(258), \ + _PyLong_DIGIT_INIT(259), \ + _PyLong_DIGIT_INIT(260), \ + _PyLong_DIGIT_INIT(261), \ + _PyLong_DIGIT_INIT(262), \ + _PyLong_DIGIT_INIT(263), \ + _PyLong_DIGIT_INIT(264), \ + _PyLong_DIGIT_INIT(265), \ + _PyLong_DIGIT_INIT(266), \ + _PyLong_DIGIT_INIT(267), \ + _PyLong_DIGIT_INIT(268), \ + _PyLong_DIGIT_INIT(269), \ + _PyLong_DIGIT_INIT(270), \ + _PyLong_DIGIT_INIT(271), \ + _PyLong_DIGIT_INIT(272), \ + _PyLong_DIGIT_INIT(273), \ + _PyLong_DIGIT_INIT(274), \ + _PyLong_DIGIT_INIT(275), \ + _PyLong_DIGIT_INIT(276), \ + _PyLong_DIGIT_INIT(277), \ + _PyLong_DIGIT_INIT(278), \ + _PyLong_DIGIT_INIT(279), \ + _PyLong_DIGIT_INIT(280), \ + _PyLong_DIGIT_INIT(281), \ + _PyLong_DIGIT_INIT(282), \ + _PyLong_DIGIT_INIT(283), \ + _PyLong_DIGIT_INIT(284), \ + _PyLong_DIGIT_INIT(285), \ + _PyLong_DIGIT_INIT(286), \ + _PyLong_DIGIT_INIT(287), \ + _PyLong_DIGIT_INIT(288), \ + _PyLong_DIGIT_INIT(289), \ + _PyLong_DIGIT_INIT(290), \ + _PyLong_DIGIT_INIT(291), \ + _PyLong_DIGIT_INIT(292), \ + _PyLong_DIGIT_INIT(293), \ + _PyLong_DIGIT_INIT(294), \ + _PyLong_DIGIT_INIT(295), \ + _PyLong_DIGIT_INIT(296), \ + _PyLong_DIGIT_INIT(297), \ + _PyLong_DIGIT_INIT(298), \ + _PyLong_DIGIT_INIT(299), \ + _PyLong_DIGIT_INIT(300), \ + _PyLong_DIGIT_INIT(301), \ + _PyLong_DIGIT_INIT(302), \ + _PyLong_DIGIT_INIT(303), \ + _PyLong_DIGIT_INIT(304), \ + _PyLong_DIGIT_INIT(305), \ + _PyLong_DIGIT_INIT(306), \ + _PyLong_DIGIT_INIT(307), \ + _PyLong_DIGIT_INIT(308), \ + _PyLong_DIGIT_INIT(309), \ + _PyLong_DIGIT_INIT(310), \ + _PyLong_DIGIT_INIT(311), \ + _PyLong_DIGIT_INIT(312), \ + _PyLong_DIGIT_INIT(313), \ + _PyLong_DIGIT_INIT(314), \ + _PyLong_DIGIT_INIT(315), \ + _PyLong_DIGIT_INIT(316), \ + _PyLong_DIGIT_INIT(317), \ + _PyLong_DIGIT_INIT(318), \ + _PyLong_DIGIT_INIT(319), \ + _PyLong_DIGIT_INIT(320), \ + _PyLong_DIGIT_INIT(321), \ + _PyLong_DIGIT_INIT(322), \ + _PyLong_DIGIT_INIT(323), \ + _PyLong_DIGIT_INIT(324), \ + _PyLong_DIGIT_INIT(325), \ + _PyLong_DIGIT_INIT(326), \ + _PyLong_DIGIT_INIT(327), \ + _PyLong_DIGIT_INIT(328), \ + _PyLong_DIGIT_INIT(329), \ + _PyLong_DIGIT_INIT(330), \ + _PyLong_DIGIT_INIT(331), \ + _PyLong_DIGIT_INIT(332), \ + _PyLong_DIGIT_INIT(333), \ + _PyLong_DIGIT_INIT(334), \ + _PyLong_DIGIT_INIT(335), \ + _PyLong_DIGIT_INIT(336), \ + _PyLong_DIGIT_INIT(337), \ + _PyLong_DIGIT_INIT(338), \ + _PyLong_DIGIT_INIT(339), \ + _PyLong_DIGIT_INIT(340), \ + _PyLong_DIGIT_INIT(341), \ + _PyLong_DIGIT_INIT(342), \ + _PyLong_DIGIT_INIT(343), \ + _PyLong_DIGIT_INIT(344), \ + _PyLong_DIGIT_INIT(345), \ + _PyLong_DIGIT_INIT(346), \ + _PyLong_DIGIT_INIT(347), \ + _PyLong_DIGIT_INIT(348), \ + _PyLong_DIGIT_INIT(349), \ + _PyLong_DIGIT_INIT(350), \ + _PyLong_DIGIT_INIT(351), \ + _PyLong_DIGIT_INIT(352), \ + _PyLong_DIGIT_INIT(353), \ + _PyLong_DIGIT_INIT(354), \ + _PyLong_DIGIT_INIT(355), \ + _PyLong_DIGIT_INIT(356), \ + _PyLong_DIGIT_INIT(357), \ + _PyLong_DIGIT_INIT(358), \ + _PyLong_DIGIT_INIT(359), \ + _PyLong_DIGIT_INIT(360), \ + _PyLong_DIGIT_INIT(361), \ + _PyLong_DIGIT_INIT(362), \ + _PyLong_DIGIT_INIT(363), \ + _PyLong_DIGIT_INIT(364), \ + _PyLong_DIGIT_INIT(365), \ + _PyLong_DIGIT_INIT(366), \ + _PyLong_DIGIT_INIT(367), \ + _PyLong_DIGIT_INIT(368), \ + _PyLong_DIGIT_INIT(369), \ + _PyLong_DIGIT_INIT(370), \ + _PyLong_DIGIT_INIT(371), \ + _PyLong_DIGIT_INIT(372), \ + _PyLong_DIGIT_INIT(373), \ + _PyLong_DIGIT_INIT(374), \ + _PyLong_DIGIT_INIT(375), \ + _PyLong_DIGIT_INIT(376), \ + _PyLong_DIGIT_INIT(377), \ + _PyLong_DIGIT_INIT(378), \ + _PyLong_DIGIT_INIT(379), \ + _PyLong_DIGIT_INIT(380), \ + _PyLong_DIGIT_INIT(381), \ + _PyLong_DIGIT_INIT(382), \ + _PyLong_DIGIT_INIT(383), \ + _PyLong_DIGIT_INIT(384), \ + _PyLong_DIGIT_INIT(385), \ + _PyLong_DIGIT_INIT(386), \ + _PyLong_DIGIT_INIT(387), \ + _PyLong_DIGIT_INIT(388), \ + _PyLong_DIGIT_INIT(389), \ + _PyLong_DIGIT_INIT(390), \ + _PyLong_DIGIT_INIT(391), \ + _PyLong_DIGIT_INIT(392), \ + _PyLong_DIGIT_INIT(393), \ + _PyLong_DIGIT_INIT(394), \ + _PyLong_DIGIT_INIT(395), \ + _PyLong_DIGIT_INIT(396), \ + _PyLong_DIGIT_INIT(397), \ + _PyLong_DIGIT_INIT(398), \ + _PyLong_DIGIT_INIT(399), \ + _PyLong_DIGIT_INIT(400), \ + _PyLong_DIGIT_INIT(401), \ + _PyLong_DIGIT_INIT(402), \ + _PyLong_DIGIT_INIT(403), \ + _PyLong_DIGIT_INIT(404), \ + _PyLong_DIGIT_INIT(405), \ + _PyLong_DIGIT_INIT(406), \ + _PyLong_DIGIT_INIT(407), \ + _PyLong_DIGIT_INIT(408), \ + _PyLong_DIGIT_INIT(409), \ + _PyLong_DIGIT_INIT(410), \ + _PyLong_DIGIT_INIT(411), \ + _PyLong_DIGIT_INIT(412), \ + _PyLong_DIGIT_INIT(413), \ + _PyLong_DIGIT_INIT(414), \ + _PyLong_DIGIT_INIT(415), \ + _PyLong_DIGIT_INIT(416), \ + _PyLong_DIGIT_INIT(417), \ + _PyLong_DIGIT_INIT(418), \ + _PyLong_DIGIT_INIT(419), \ + _PyLong_DIGIT_INIT(420), \ + _PyLong_DIGIT_INIT(421), \ + _PyLong_DIGIT_INIT(422), \ + _PyLong_DIGIT_INIT(423), \ + _PyLong_DIGIT_INIT(424), \ + _PyLong_DIGIT_INIT(425), \ + _PyLong_DIGIT_INIT(426), \ + _PyLong_DIGIT_INIT(427), \ + _PyLong_DIGIT_INIT(428), \ + _PyLong_DIGIT_INIT(429), \ + _PyLong_DIGIT_INIT(430), \ + _PyLong_DIGIT_INIT(431), \ + _PyLong_DIGIT_INIT(432), \ + _PyLong_DIGIT_INIT(433), \ + _PyLong_DIGIT_INIT(434), \ + _PyLong_DIGIT_INIT(435), \ + _PyLong_DIGIT_INIT(436), \ + _PyLong_DIGIT_INIT(437), \ + _PyLong_DIGIT_INIT(438), \ + _PyLong_DIGIT_INIT(439), \ + _PyLong_DIGIT_INIT(440), \ + _PyLong_DIGIT_INIT(441), \ + _PyLong_DIGIT_INIT(442), \ + _PyLong_DIGIT_INIT(443), \ + _PyLong_DIGIT_INIT(444), \ + _PyLong_DIGIT_INIT(445), \ + _PyLong_DIGIT_INIT(446), \ + _PyLong_DIGIT_INIT(447), \ + _PyLong_DIGIT_INIT(448), \ + _PyLong_DIGIT_INIT(449), \ + _PyLong_DIGIT_INIT(450), \ + _PyLong_DIGIT_INIT(451), \ + _PyLong_DIGIT_INIT(452), \ + _PyLong_DIGIT_INIT(453), \ + _PyLong_DIGIT_INIT(454), \ + _PyLong_DIGIT_INIT(455), \ + _PyLong_DIGIT_INIT(456), \ + _PyLong_DIGIT_INIT(457), \ + _PyLong_DIGIT_INIT(458), \ + _PyLong_DIGIT_INIT(459), \ + _PyLong_DIGIT_INIT(460), \ + _PyLong_DIGIT_INIT(461), \ + _PyLong_DIGIT_INIT(462), \ + _PyLong_DIGIT_INIT(463), \ + _PyLong_DIGIT_INIT(464), \ + _PyLong_DIGIT_INIT(465), \ + _PyLong_DIGIT_INIT(466), \ + _PyLong_DIGIT_INIT(467), \ + _PyLong_DIGIT_INIT(468), \ + _PyLong_DIGIT_INIT(469), \ + _PyLong_DIGIT_INIT(470), \ + _PyLong_DIGIT_INIT(471), \ + _PyLong_DIGIT_INIT(472), \ + _PyLong_DIGIT_INIT(473), \ + _PyLong_DIGIT_INIT(474), \ + _PyLong_DIGIT_INIT(475), \ + _PyLong_DIGIT_INIT(476), \ + _PyLong_DIGIT_INIT(477), \ + _PyLong_DIGIT_INIT(478), \ + _PyLong_DIGIT_INIT(479), \ + _PyLong_DIGIT_INIT(480), \ + _PyLong_DIGIT_INIT(481), \ + _PyLong_DIGIT_INIT(482), \ + _PyLong_DIGIT_INIT(483), \ + _PyLong_DIGIT_INIT(484), \ + _PyLong_DIGIT_INIT(485), \ + _PyLong_DIGIT_INIT(486), \ + _PyLong_DIGIT_INIT(487), \ + _PyLong_DIGIT_INIT(488), \ + _PyLong_DIGIT_INIT(489), \ + _PyLong_DIGIT_INIT(490), \ + _PyLong_DIGIT_INIT(491), \ + _PyLong_DIGIT_INIT(492), \ + _PyLong_DIGIT_INIT(493), \ + _PyLong_DIGIT_INIT(494), \ + _PyLong_DIGIT_INIT(495), \ + _PyLong_DIGIT_INIT(496), \ + _PyLong_DIGIT_INIT(497), \ + _PyLong_DIGIT_INIT(498), \ + _PyLong_DIGIT_INIT(499), \ + _PyLong_DIGIT_INIT(500), \ + _PyLong_DIGIT_INIT(501), \ + _PyLong_DIGIT_INIT(502), \ + _PyLong_DIGIT_INIT(503), \ + _PyLong_DIGIT_INIT(504), \ + _PyLong_DIGIT_INIT(505), \ + _PyLong_DIGIT_INIT(506), \ + _PyLong_DIGIT_INIT(507), \ + _PyLong_DIGIT_INIT(508), \ + _PyLong_DIGIT_INIT(509), \ + _PyLong_DIGIT_INIT(510), \ + _PyLong_DIGIT_INIT(511), \ + _PyLong_DIGIT_INIT(512), \ + _PyLong_DIGIT_INIT(513), \ + _PyLong_DIGIT_INIT(514), \ + _PyLong_DIGIT_INIT(515), \ + _PyLong_DIGIT_INIT(516), \ + _PyLong_DIGIT_INIT(517), \ + _PyLong_DIGIT_INIT(518), \ + _PyLong_DIGIT_INIT(519), \ + _PyLong_DIGIT_INIT(520), \ + _PyLong_DIGIT_INIT(521), \ + _PyLong_DIGIT_INIT(522), \ + _PyLong_DIGIT_INIT(523), \ + _PyLong_DIGIT_INIT(524), \ + _PyLong_DIGIT_INIT(525), \ + _PyLong_DIGIT_INIT(526), \ + _PyLong_DIGIT_INIT(527), \ + _PyLong_DIGIT_INIT(528), \ + _PyLong_DIGIT_INIT(529), \ + _PyLong_DIGIT_INIT(530), \ + _PyLong_DIGIT_INIT(531), \ + _PyLong_DIGIT_INIT(532), \ + _PyLong_DIGIT_INIT(533), \ + _PyLong_DIGIT_INIT(534), \ + _PyLong_DIGIT_INIT(535), \ + _PyLong_DIGIT_INIT(536), \ + _PyLong_DIGIT_INIT(537), \ + _PyLong_DIGIT_INIT(538), \ + _PyLong_DIGIT_INIT(539), \ + _PyLong_DIGIT_INIT(540), \ + _PyLong_DIGIT_INIT(541), \ + _PyLong_DIGIT_INIT(542), \ + _PyLong_DIGIT_INIT(543), \ + _PyLong_DIGIT_INIT(544), \ + _PyLong_DIGIT_INIT(545), \ + _PyLong_DIGIT_INIT(546), \ + _PyLong_DIGIT_INIT(547), \ + _PyLong_DIGIT_INIT(548), \ + _PyLong_DIGIT_INIT(549), \ + _PyLong_DIGIT_INIT(550), \ + _PyLong_DIGIT_INIT(551), \ + _PyLong_DIGIT_INIT(552), \ + _PyLong_DIGIT_INIT(553), \ + _PyLong_DIGIT_INIT(554), \ + _PyLong_DIGIT_INIT(555), \ + _PyLong_DIGIT_INIT(556), \ + _PyLong_DIGIT_INIT(557), \ + _PyLong_DIGIT_INIT(558), \ + _PyLong_DIGIT_INIT(559), \ + _PyLong_DIGIT_INIT(560), \ + _PyLong_DIGIT_INIT(561), \ + _PyLong_DIGIT_INIT(562), \ + _PyLong_DIGIT_INIT(563), \ + _PyLong_DIGIT_INIT(564), \ + _PyLong_DIGIT_INIT(565), \ + _PyLong_DIGIT_INIT(566), \ + _PyLong_DIGIT_INIT(567), \ + _PyLong_DIGIT_INIT(568), \ + _PyLong_DIGIT_INIT(569), \ + _PyLong_DIGIT_INIT(570), \ + _PyLong_DIGIT_INIT(571), \ + _PyLong_DIGIT_INIT(572), \ + _PyLong_DIGIT_INIT(573), \ + _PyLong_DIGIT_INIT(574), \ + _PyLong_DIGIT_INIT(575), \ + _PyLong_DIGIT_INIT(576), \ + _PyLong_DIGIT_INIT(577), \ + _PyLong_DIGIT_INIT(578), \ + _PyLong_DIGIT_INIT(579), \ + _PyLong_DIGIT_INIT(580), \ + _PyLong_DIGIT_INIT(581), \ + _PyLong_DIGIT_INIT(582), \ + _PyLong_DIGIT_INIT(583), \ + _PyLong_DIGIT_INIT(584), \ + _PyLong_DIGIT_INIT(585), \ + _PyLong_DIGIT_INIT(586), \ + _PyLong_DIGIT_INIT(587), \ + _PyLong_DIGIT_INIT(588), \ + _PyLong_DIGIT_INIT(589), \ + _PyLong_DIGIT_INIT(590), \ + _PyLong_DIGIT_INIT(591), \ + _PyLong_DIGIT_INIT(592), \ + _PyLong_DIGIT_INIT(593), \ + _PyLong_DIGIT_INIT(594), \ + _PyLong_DIGIT_INIT(595), \ + _PyLong_DIGIT_INIT(596), \ + _PyLong_DIGIT_INIT(597), \ + _PyLong_DIGIT_INIT(598), \ + _PyLong_DIGIT_INIT(599), \ + _PyLong_DIGIT_INIT(600), \ + _PyLong_DIGIT_INIT(601), \ + _PyLong_DIGIT_INIT(602), \ + _PyLong_DIGIT_INIT(603), \ + _PyLong_DIGIT_INIT(604), \ + _PyLong_DIGIT_INIT(605), \ + _PyLong_DIGIT_INIT(606), \ + _PyLong_DIGIT_INIT(607), \ + _PyLong_DIGIT_INIT(608), \ + _PyLong_DIGIT_INIT(609), \ + _PyLong_DIGIT_INIT(610), \ + _PyLong_DIGIT_INIT(611), \ + _PyLong_DIGIT_INIT(612), \ + _PyLong_DIGIT_INIT(613), \ + _PyLong_DIGIT_INIT(614), \ + _PyLong_DIGIT_INIT(615), \ + _PyLong_DIGIT_INIT(616), \ + _PyLong_DIGIT_INIT(617), \ + _PyLong_DIGIT_INIT(618), \ + _PyLong_DIGIT_INIT(619), \ + _PyLong_DIGIT_INIT(620), \ + _PyLong_DIGIT_INIT(621), \ + _PyLong_DIGIT_INIT(622), \ + _PyLong_DIGIT_INIT(623), \ + _PyLong_DIGIT_INIT(624), \ + _PyLong_DIGIT_INIT(625), \ + _PyLong_DIGIT_INIT(626), \ + _PyLong_DIGIT_INIT(627), \ + _PyLong_DIGIT_INIT(628), \ + _PyLong_DIGIT_INIT(629), \ + _PyLong_DIGIT_INIT(630), \ + _PyLong_DIGIT_INIT(631), \ + _PyLong_DIGIT_INIT(632), \ + _PyLong_DIGIT_INIT(633), \ + _PyLong_DIGIT_INIT(634), \ + _PyLong_DIGIT_INIT(635), \ + _PyLong_DIGIT_INIT(636), \ + _PyLong_DIGIT_INIT(637), \ + _PyLong_DIGIT_INIT(638), \ + _PyLong_DIGIT_INIT(639), \ + _PyLong_DIGIT_INIT(640), \ + _PyLong_DIGIT_INIT(641), \ + _PyLong_DIGIT_INIT(642), \ + _PyLong_DIGIT_INIT(643), \ + _PyLong_DIGIT_INIT(644), \ + _PyLong_DIGIT_INIT(645), \ + _PyLong_DIGIT_INIT(646), \ + _PyLong_DIGIT_INIT(647), \ + _PyLong_DIGIT_INIT(648), \ + _PyLong_DIGIT_INIT(649), \ + _PyLong_DIGIT_INIT(650), \ + _PyLong_DIGIT_INIT(651), \ + _PyLong_DIGIT_INIT(652), \ + _PyLong_DIGIT_INIT(653), \ + _PyLong_DIGIT_INIT(654), \ + _PyLong_DIGIT_INIT(655), \ + _PyLong_DIGIT_INIT(656), \ + _PyLong_DIGIT_INIT(657), \ + _PyLong_DIGIT_INIT(658), \ + _PyLong_DIGIT_INIT(659), \ + _PyLong_DIGIT_INIT(660), \ + _PyLong_DIGIT_INIT(661), \ + _PyLong_DIGIT_INIT(662), \ + _PyLong_DIGIT_INIT(663), \ + _PyLong_DIGIT_INIT(664), \ + _PyLong_DIGIT_INIT(665), \ + _PyLong_DIGIT_INIT(666), \ + _PyLong_DIGIT_INIT(667), \ + _PyLong_DIGIT_INIT(668), \ + _PyLong_DIGIT_INIT(669), \ + _PyLong_DIGIT_INIT(670), \ + _PyLong_DIGIT_INIT(671), \ + _PyLong_DIGIT_INIT(672), \ + _PyLong_DIGIT_INIT(673), \ + _PyLong_DIGIT_INIT(674), \ + _PyLong_DIGIT_INIT(675), \ + _PyLong_DIGIT_INIT(676), \ + _PyLong_DIGIT_INIT(677), \ + _PyLong_DIGIT_INIT(678), \ + _PyLong_DIGIT_INIT(679), \ + _PyLong_DIGIT_INIT(680), \ + _PyLong_DIGIT_INIT(681), \ + _PyLong_DIGIT_INIT(682), \ + _PyLong_DIGIT_INIT(683), \ + _PyLong_DIGIT_INIT(684), \ + _PyLong_DIGIT_INIT(685), \ + _PyLong_DIGIT_INIT(686), \ + _PyLong_DIGIT_INIT(687), \ + _PyLong_DIGIT_INIT(688), \ + _PyLong_DIGIT_INIT(689), \ + _PyLong_DIGIT_INIT(690), \ + _PyLong_DIGIT_INIT(691), \ + _PyLong_DIGIT_INIT(692), \ + _PyLong_DIGIT_INIT(693), \ + _PyLong_DIGIT_INIT(694), \ + _PyLong_DIGIT_INIT(695), \ + _PyLong_DIGIT_INIT(696), \ + _PyLong_DIGIT_INIT(697), \ + _PyLong_DIGIT_INIT(698), \ + _PyLong_DIGIT_INIT(699), \ + _PyLong_DIGIT_INIT(700), \ + _PyLong_DIGIT_INIT(701), \ + _PyLong_DIGIT_INIT(702), \ + _PyLong_DIGIT_INIT(703), \ + _PyLong_DIGIT_INIT(704), \ + _PyLong_DIGIT_INIT(705), \ + _PyLong_DIGIT_INIT(706), \ + _PyLong_DIGIT_INIT(707), \ + _PyLong_DIGIT_INIT(708), \ + _PyLong_DIGIT_INIT(709), \ + _PyLong_DIGIT_INIT(710), \ + _PyLong_DIGIT_INIT(711), \ + _PyLong_DIGIT_INIT(712), \ + _PyLong_DIGIT_INIT(713), \ + _PyLong_DIGIT_INIT(714), \ + _PyLong_DIGIT_INIT(715), \ + _PyLong_DIGIT_INIT(716), \ + _PyLong_DIGIT_INIT(717), \ + _PyLong_DIGIT_INIT(718), \ + _PyLong_DIGIT_INIT(719), \ + _PyLong_DIGIT_INIT(720), \ + _PyLong_DIGIT_INIT(721), \ + _PyLong_DIGIT_INIT(722), \ + _PyLong_DIGIT_INIT(723), \ + _PyLong_DIGIT_INIT(724), \ + _PyLong_DIGIT_INIT(725), \ + _PyLong_DIGIT_INIT(726), \ + _PyLong_DIGIT_INIT(727), \ + _PyLong_DIGIT_INIT(728), \ + _PyLong_DIGIT_INIT(729), \ + _PyLong_DIGIT_INIT(730), \ + _PyLong_DIGIT_INIT(731), \ + _PyLong_DIGIT_INIT(732), \ + _PyLong_DIGIT_INIT(733), \ + _PyLong_DIGIT_INIT(734), \ + _PyLong_DIGIT_INIT(735), \ + _PyLong_DIGIT_INIT(736), \ + _PyLong_DIGIT_INIT(737), \ + _PyLong_DIGIT_INIT(738), \ + _PyLong_DIGIT_INIT(739), \ + _PyLong_DIGIT_INIT(740), \ + _PyLong_DIGIT_INIT(741), \ + _PyLong_DIGIT_INIT(742), \ + _PyLong_DIGIT_INIT(743), \ + _PyLong_DIGIT_INIT(744), \ + _PyLong_DIGIT_INIT(745), \ + _PyLong_DIGIT_INIT(746), \ + _PyLong_DIGIT_INIT(747), \ + _PyLong_DIGIT_INIT(748), \ + _PyLong_DIGIT_INIT(749), \ + _PyLong_DIGIT_INIT(750), \ + _PyLong_DIGIT_INIT(751), \ + _PyLong_DIGIT_INIT(752), \ + _PyLong_DIGIT_INIT(753), \ + _PyLong_DIGIT_INIT(754), \ + _PyLong_DIGIT_INIT(755), \ + _PyLong_DIGIT_INIT(756), \ + _PyLong_DIGIT_INIT(757), \ + _PyLong_DIGIT_INIT(758), \ + _PyLong_DIGIT_INIT(759), \ + _PyLong_DIGIT_INIT(760), \ + _PyLong_DIGIT_INIT(761), \ + _PyLong_DIGIT_INIT(762), \ + _PyLong_DIGIT_INIT(763), \ + _PyLong_DIGIT_INIT(764), \ + _PyLong_DIGIT_INIT(765), \ + _PyLong_DIGIT_INIT(766), \ + _PyLong_DIGIT_INIT(767), \ + _PyLong_DIGIT_INIT(768), \ + _PyLong_DIGIT_INIT(769), \ + _PyLong_DIGIT_INIT(770), \ + _PyLong_DIGIT_INIT(771), \ + _PyLong_DIGIT_INIT(772), \ + _PyLong_DIGIT_INIT(773), \ + _PyLong_DIGIT_INIT(774), \ + _PyLong_DIGIT_INIT(775), \ + _PyLong_DIGIT_INIT(776), \ + _PyLong_DIGIT_INIT(777), \ + _PyLong_DIGIT_INIT(778), \ + _PyLong_DIGIT_INIT(779), \ + _PyLong_DIGIT_INIT(780), \ + _PyLong_DIGIT_INIT(781), \ + _PyLong_DIGIT_INIT(782), \ + _PyLong_DIGIT_INIT(783), \ + _PyLong_DIGIT_INIT(784), \ + _PyLong_DIGIT_INIT(785), \ + _PyLong_DIGIT_INIT(786), \ + _PyLong_DIGIT_INIT(787), \ + _PyLong_DIGIT_INIT(788), \ + _PyLong_DIGIT_INIT(789), \ + _PyLong_DIGIT_INIT(790), \ + _PyLong_DIGIT_INIT(791), \ + _PyLong_DIGIT_INIT(792), \ + _PyLong_DIGIT_INIT(793), \ + _PyLong_DIGIT_INIT(794), \ + _PyLong_DIGIT_INIT(795), \ + _PyLong_DIGIT_INIT(796), \ + _PyLong_DIGIT_INIT(797), \ + _PyLong_DIGIT_INIT(798), \ + _PyLong_DIGIT_INIT(799), \ + _PyLong_DIGIT_INIT(800), \ + _PyLong_DIGIT_INIT(801), \ + _PyLong_DIGIT_INIT(802), \ + _PyLong_DIGIT_INIT(803), \ + _PyLong_DIGIT_INIT(804), \ + _PyLong_DIGIT_INIT(805), \ + _PyLong_DIGIT_INIT(806), \ + _PyLong_DIGIT_INIT(807), \ + _PyLong_DIGIT_INIT(808), \ + _PyLong_DIGIT_INIT(809), \ + _PyLong_DIGIT_INIT(810), \ + _PyLong_DIGIT_INIT(811), \ + _PyLong_DIGIT_INIT(812), \ + _PyLong_DIGIT_INIT(813), \ + _PyLong_DIGIT_INIT(814), \ + _PyLong_DIGIT_INIT(815), \ + _PyLong_DIGIT_INIT(816), \ + _PyLong_DIGIT_INIT(817), \ + _PyLong_DIGIT_INIT(818), \ + _PyLong_DIGIT_INIT(819), \ + _PyLong_DIGIT_INIT(820), \ + _PyLong_DIGIT_INIT(821), \ + _PyLong_DIGIT_INIT(822), \ + _PyLong_DIGIT_INIT(823), \ + _PyLong_DIGIT_INIT(824), \ + _PyLong_DIGIT_INIT(825), \ + _PyLong_DIGIT_INIT(826), \ + _PyLong_DIGIT_INIT(827), \ + _PyLong_DIGIT_INIT(828), \ + _PyLong_DIGIT_INIT(829), \ + _PyLong_DIGIT_INIT(830), \ + _PyLong_DIGIT_INIT(831), \ + _PyLong_DIGIT_INIT(832), \ + _PyLong_DIGIT_INIT(833), \ + _PyLong_DIGIT_INIT(834), \ + _PyLong_DIGIT_INIT(835), \ + _PyLong_DIGIT_INIT(836), \ + _PyLong_DIGIT_INIT(837), \ + _PyLong_DIGIT_INIT(838), \ + _PyLong_DIGIT_INIT(839), \ + _PyLong_DIGIT_INIT(840), \ + _PyLong_DIGIT_INIT(841), \ + _PyLong_DIGIT_INIT(842), \ + _PyLong_DIGIT_INIT(843), \ + _PyLong_DIGIT_INIT(844), \ + _PyLong_DIGIT_INIT(845), \ + _PyLong_DIGIT_INIT(846), \ + _PyLong_DIGIT_INIT(847), \ + _PyLong_DIGIT_INIT(848), \ + _PyLong_DIGIT_INIT(849), \ + _PyLong_DIGIT_INIT(850), \ + _PyLong_DIGIT_INIT(851), \ + _PyLong_DIGIT_INIT(852), \ + _PyLong_DIGIT_INIT(853), \ + _PyLong_DIGIT_INIT(854), \ + _PyLong_DIGIT_INIT(855), \ + _PyLong_DIGIT_INIT(856), \ + _PyLong_DIGIT_INIT(857), \ + _PyLong_DIGIT_INIT(858), \ + _PyLong_DIGIT_INIT(859), \ + _PyLong_DIGIT_INIT(860), \ + _PyLong_DIGIT_INIT(861), \ + _PyLong_DIGIT_INIT(862), \ + _PyLong_DIGIT_INIT(863), \ + _PyLong_DIGIT_INIT(864), \ + _PyLong_DIGIT_INIT(865), \ + _PyLong_DIGIT_INIT(866), \ + _PyLong_DIGIT_INIT(867), \ + _PyLong_DIGIT_INIT(868), \ + _PyLong_DIGIT_INIT(869), \ + _PyLong_DIGIT_INIT(870), \ + _PyLong_DIGIT_INIT(871), \ + _PyLong_DIGIT_INIT(872), \ + _PyLong_DIGIT_INIT(873), \ + _PyLong_DIGIT_INIT(874), \ + _PyLong_DIGIT_INIT(875), \ + _PyLong_DIGIT_INIT(876), \ + _PyLong_DIGIT_INIT(877), \ + _PyLong_DIGIT_INIT(878), \ + _PyLong_DIGIT_INIT(879), \ + _PyLong_DIGIT_INIT(880), \ + _PyLong_DIGIT_INIT(881), \ + _PyLong_DIGIT_INIT(882), \ + _PyLong_DIGIT_INIT(883), \ + _PyLong_DIGIT_INIT(884), \ + _PyLong_DIGIT_INIT(885), \ + _PyLong_DIGIT_INIT(886), \ + _PyLong_DIGIT_INIT(887), \ + _PyLong_DIGIT_INIT(888), \ + _PyLong_DIGIT_INIT(889), \ + _PyLong_DIGIT_INIT(890), \ + _PyLong_DIGIT_INIT(891), \ + _PyLong_DIGIT_INIT(892), \ + _PyLong_DIGIT_INIT(893), \ + _PyLong_DIGIT_INIT(894), \ + _PyLong_DIGIT_INIT(895), \ + _PyLong_DIGIT_INIT(896), \ + _PyLong_DIGIT_INIT(897), \ + _PyLong_DIGIT_INIT(898), \ + _PyLong_DIGIT_INIT(899), \ + _PyLong_DIGIT_INIT(900), \ + _PyLong_DIGIT_INIT(901), \ + _PyLong_DIGIT_INIT(902), \ + _PyLong_DIGIT_INIT(903), \ + _PyLong_DIGIT_INIT(904), \ + _PyLong_DIGIT_INIT(905), \ + _PyLong_DIGIT_INIT(906), \ + _PyLong_DIGIT_INIT(907), \ + _PyLong_DIGIT_INIT(908), \ + _PyLong_DIGIT_INIT(909), \ + _PyLong_DIGIT_INIT(910), \ + _PyLong_DIGIT_INIT(911), \ + _PyLong_DIGIT_INIT(912), \ + _PyLong_DIGIT_INIT(913), \ + _PyLong_DIGIT_INIT(914), \ + _PyLong_DIGIT_INIT(915), \ + _PyLong_DIGIT_INIT(916), \ + _PyLong_DIGIT_INIT(917), \ + _PyLong_DIGIT_INIT(918), \ + _PyLong_DIGIT_INIT(919), \ + _PyLong_DIGIT_INIT(920), \ + _PyLong_DIGIT_INIT(921), \ + _PyLong_DIGIT_INIT(922), \ + _PyLong_DIGIT_INIT(923), \ + _PyLong_DIGIT_INIT(924), \ + _PyLong_DIGIT_INIT(925), \ + _PyLong_DIGIT_INIT(926), \ + _PyLong_DIGIT_INIT(927), \ + _PyLong_DIGIT_INIT(928), \ + _PyLong_DIGIT_INIT(929), \ + _PyLong_DIGIT_INIT(930), \ + _PyLong_DIGIT_INIT(931), \ + _PyLong_DIGIT_INIT(932), \ + _PyLong_DIGIT_INIT(933), \ + _PyLong_DIGIT_INIT(934), \ + _PyLong_DIGIT_INIT(935), \ + _PyLong_DIGIT_INIT(936), \ + _PyLong_DIGIT_INIT(937), \ + _PyLong_DIGIT_INIT(938), \ + _PyLong_DIGIT_INIT(939), \ + _PyLong_DIGIT_INIT(940), \ + _PyLong_DIGIT_INIT(941), \ + _PyLong_DIGIT_INIT(942), \ + _PyLong_DIGIT_INIT(943), \ + _PyLong_DIGIT_INIT(944), \ + _PyLong_DIGIT_INIT(945), \ + _PyLong_DIGIT_INIT(946), \ + _PyLong_DIGIT_INIT(947), \ + _PyLong_DIGIT_INIT(948), \ + _PyLong_DIGIT_INIT(949), \ + _PyLong_DIGIT_INIT(950), \ + _PyLong_DIGIT_INIT(951), \ + _PyLong_DIGIT_INIT(952), \ + _PyLong_DIGIT_INIT(953), \ + _PyLong_DIGIT_INIT(954), \ + _PyLong_DIGIT_INIT(955), \ + _PyLong_DIGIT_INIT(956), \ + _PyLong_DIGIT_INIT(957), \ + _PyLong_DIGIT_INIT(958), \ + _PyLong_DIGIT_INIT(959), \ + _PyLong_DIGIT_INIT(960), \ + _PyLong_DIGIT_INIT(961), \ + _PyLong_DIGIT_INIT(962), \ + _PyLong_DIGIT_INIT(963), \ + _PyLong_DIGIT_INIT(964), \ + _PyLong_DIGIT_INIT(965), \ + _PyLong_DIGIT_INIT(966), \ + _PyLong_DIGIT_INIT(967), \ + _PyLong_DIGIT_INIT(968), \ + _PyLong_DIGIT_INIT(969), \ + _PyLong_DIGIT_INIT(970), \ + _PyLong_DIGIT_INIT(971), \ + _PyLong_DIGIT_INIT(972), \ + _PyLong_DIGIT_INIT(973), \ + _PyLong_DIGIT_INIT(974), \ + _PyLong_DIGIT_INIT(975), \ + _PyLong_DIGIT_INIT(976), \ + _PyLong_DIGIT_INIT(977), \ + _PyLong_DIGIT_INIT(978), \ + _PyLong_DIGIT_INIT(979), \ + _PyLong_DIGIT_INIT(980), \ + _PyLong_DIGIT_INIT(981), \ + _PyLong_DIGIT_INIT(982), \ + _PyLong_DIGIT_INIT(983), \ + _PyLong_DIGIT_INIT(984), \ + _PyLong_DIGIT_INIT(985), \ + _PyLong_DIGIT_INIT(986), \ + _PyLong_DIGIT_INIT(987), \ + _PyLong_DIGIT_INIT(988), \ + _PyLong_DIGIT_INIT(989), \ + _PyLong_DIGIT_INIT(990), \ + _PyLong_DIGIT_INIT(991), \ + _PyLong_DIGIT_INIT(992), \ + _PyLong_DIGIT_INIT(993), \ + _PyLong_DIGIT_INIT(994), \ + _PyLong_DIGIT_INIT(995), \ + _PyLong_DIGIT_INIT(996), \ + _PyLong_DIGIT_INIT(997), \ + _PyLong_DIGIT_INIT(998), \ + _PyLong_DIGIT_INIT(999), \ + _PyLong_DIGIT_INIT(1000), \ + _PyLong_DIGIT_INIT(1001), \ + _PyLong_DIGIT_INIT(1002), \ + _PyLong_DIGIT_INIT(1003), \ + _PyLong_DIGIT_INIT(1004), \ + _PyLong_DIGIT_INIT(1005), \ + _PyLong_DIGIT_INIT(1006), \ + _PyLong_DIGIT_INIT(1007), \ + _PyLong_DIGIT_INIT(1008), \ + _PyLong_DIGIT_INIT(1009), \ + _PyLong_DIGIT_INIT(1010), \ + _PyLong_DIGIT_INIT(1011), \ + _PyLong_DIGIT_INIT(1012), \ + _PyLong_DIGIT_INIT(1013), \ + _PyLong_DIGIT_INIT(1014), \ + _PyLong_DIGIT_INIT(1015), \ + _PyLong_DIGIT_INIT(1016), \ + _PyLong_DIGIT_INIT(1017), \ + _PyLong_DIGIT_INIT(1018), \ + _PyLong_DIGIT_INIT(1019), \ + _PyLong_DIGIT_INIT(1020), \ + _PyLong_DIGIT_INIT(1021), \ + _PyLong_DIGIT_INIT(1022), \ + _PyLong_DIGIT_INIT(1023), \ + _PyLong_DIGIT_INIT(1024), \ } #define _Py_bytes_characters_INIT { \ @@ -553,19 +1321,35 @@ extern "C" { INIT_STR(dot_locals, "."), \ INIT_STR(empty, ""), \ INIT_STR(format, ".format"), \ + INIT_STR(gc, ""), \ INIT_STR(generic_base, ".generic_base"), \ INIT_STR(json_decoder, "json.decoder"), \ INIT_STR(kwdefaults, ".kwdefaults"), \ INIT_STR(list_err, "list index out of range"), \ + INIT_STR(native, ""), \ INIT_STR(str_replace_inf, "1e309"), \ INIT_STR(type_params, ".type_params"), \ INIT_STR(utf_8, "utf-8"), \ } #define _Py_str_identifiers_INIT { \ + INIT_ID(AGEN_CLOSED), \ + INIT_ID(AGEN_CREATED), \ + INIT_ID(AGEN_RUNNING), \ + INIT_ID(AGEN_SUSPENDED), \ INIT_ID(CANCELLED), \ + INIT_ID(CORO_CLOSED), \ + INIT_ID(CORO_CREATED), \ + INIT_ID(CORO_RUNNING), \ + INIT_ID(CORO_SUSPENDED), \ + INIT_ID(Emax), \ + INIT_ID(Emin), \ INIT_ID(FINISHED), \ INIT_ID(False), \ + INIT_ID(GEN_CLOSED), \ + INIT_ID(GEN_CREATED), \ + INIT_ID(GEN_RUNNING), \ + INIT_ID(GEN_SUSPENDED), \ INIT_ID(JSONDecodeError), \ INIT_ID(PENDING), \ INIT_ID(Py_Repr), \ @@ -660,6 +1444,8 @@ extern "C" { INIT_ID(__iter__), \ INIT_ID(__itruediv__), \ INIT_ID(__ixor__), \ + INIT_ID(__lazy_import__), \ + INIT_ID(__lazy_modules__), \ INIT_ID(__le__), \ INIT_ID(__len__), \ INIT_ID(__length_hint__), \ @@ -788,13 +1574,16 @@ extern "C" { INIT_ID(aclose), \ INIT_ID(add), \ INIT_ID(add_done_callback), \ + INIT_ID(adobe), \ INIT_ID(after_in_child), \ INIT_ID(after_in_parent), \ INIT_ID(alias), \ INIT_ID(align), \ INIT_ID(all), \ + INIT_ID(all_interpreters), \ INIT_ID(all_threads), \ INIT_ID(allow_code), \ + INIT_ID(alphabet), \ INIT_ID(any), \ INIT_ID(append), \ INIT_ID(arg), \ @@ -816,6 +1605,7 @@ extern "C" { INIT_ID(bit_offset), \ INIT_ID(bit_size), \ INIT_ID(block), \ + INIT_ID(blocking), \ INIT_ID(bound), \ INIT_ID(buffer), \ INIT_ID(buffer_callback), \ @@ -833,6 +1623,8 @@ extern "C" { INIT_ID(c_exception), \ INIT_ID(c_parameter_type), \ INIT_ID(c_return), \ + INIT_ID(c_stack), \ + INIT_ID(cache_frames), \ INIT_ID(cached_datetime_module), \ INIT_ID(cached_statements), \ INIT_ID(cadata), \ @@ -840,13 +1632,18 @@ extern "C" { INIT_ID(call), \ INIT_ID(call_exception_handler), \ INIT_ID(call_soon), \ + INIT_ID(callable), \ INIT_ID(callback), \ INIT_ID(cancel), \ + INIT_ID(canonical), \ INIT_ID(capath), \ + INIT_ID(capitals), \ INIT_ID(category), \ INIT_ID(cb_type), \ INIT_ID(certfile), \ + INIT_ID(chain), \ INIT_ID(check_same_thread), \ + INIT_ID(clamp), \ INIT_ID(clear), \ INIT_ID(close), \ INIT_ID(closed), \ @@ -872,9 +1669,12 @@ extern "C" { INIT_ID(co_varnames), \ INIT_ID(code), \ INIT_ID(col_offset), \ + INIT_ID(collector), \ INIT_ID(command), \ INIT_ID(comment_factory), \ INIT_ID(compile_mode), \ + INIT_ID(compression), \ + INIT_ID(config), \ INIT_ID(consts), \ INIT_ID(context), \ INIT_ID(contravariant), \ @@ -885,11 +1685,14 @@ extern "C" { INIT_ID(coro), \ INIT_ID(count), \ INIT_ID(covariant), \ + INIT_ID(ctx), \ INIT_ID(cwd), \ INIT_ID(d_parameter_type), \ INIT_ID(data), \ INIT_ID(database), \ + INIT_ID(date), \ INIT_ID(day), \ + INIT_ID(days), \ INIT_ID(debug), \ INIT_ID(decode), \ INIT_ID(decoder), \ @@ -919,6 +1722,7 @@ extern "C" { INIT_ID(eager_start), \ INIT_ID(effective_ids), \ INIT_ID(element_factory), \ + INIT_ID(emptyerror), \ INIT_ID(encode), \ INIT_ID(encoding), \ INIT_ID(end), \ @@ -931,17 +1735,22 @@ extern "C" { INIT_ID(errors), \ INIT_ID(event), \ INIT_ID(eventmask), \ + INIT_ID(exc), \ + INIT_ID(exc_tb), \ INIT_ID(exc_type), \ + INIT_ID(exc_val), \ INIT_ID(exc_value), \ INIT_ID(excepthook), \ INIT_ID(exception), \ INIT_ID(existing_file_name), \ + INIT_ID(exit), \ INIT_ID(exp), \ INIT_ID(expression), \ INIT_ID(extend), \ INIT_ID(extra_tokens), \ INIT_ID(facility), \ INIT_ID(factory), \ + INIT_ID(fallback), \ INIT_ID(false), \ INIT_ID(family), \ INIT_ID(fanout), \ @@ -964,17 +1773,21 @@ extern "C" { INIT_ID(flags), \ INIT_ID(flush), \ INIT_ID(fold), \ + INIT_ID(foldspaces), \ INIT_ID(follow_symlinks), \ INIT_ID(format), \ INIT_ID(format_spec), \ INIT_ID(frame_buffer), \ + INIT_ID(free_threaded), \ INIT_ID(from_param), \ INIT_ID(fromlist), \ INIT_ID(fromtimestamp), \ INIT_ID(fromutc), \ INIT_ID(fset), \ + INIT_ID(fullerror), \ INIT_ID(func), \ INIT_ID(future), \ + INIT_ID(gc), \ INIT_ID(generation), \ INIT_ID(get), \ INIT_ID(get_debug), \ @@ -996,13 +1809,17 @@ extern "C" { INIT_ID(hi), \ INIT_ID(hook), \ INIT_ID(hour), \ + INIT_ID(hours), \ INIT_ID(id), \ INIT_ID(ident), \ INIT_ID(identity_hint), \ INIT_ID(ignore), \ + INIT_ID(ignorechars), \ INIT_ID(imag), \ + INIT_ID(implieslink), \ INIT_ID(importlib), \ INIT_ID(in_fd), \ + INIT_ID(include_aliases), \ INIT_ID(incoming), \ INIT_ID(index), \ INIT_ID(indexgroup), \ @@ -1049,6 +1866,7 @@ extern "C" { INIT_ID(kw), \ INIT_ID(kw1), \ INIT_ID(kw2), \ + INIT_ID(kwargs), \ INIT_ID(kwdefaults), \ INIT_ID(label), \ INIT_ID(last), \ @@ -1059,6 +1877,7 @@ extern "C" { INIT_ID(last_value), \ INIT_ID(latin1), \ INIT_ID(leaf_size), \ + INIT_ID(legacy), \ INIT_ID(len), \ INIT_ID(length), \ INIT_ID(level), \ @@ -1074,12 +1893,15 @@ extern "C" { INIT_ID(loop), \ INIT_ID(manual_reset), \ INIT_ID(mapping), \ + INIT_ID(mask), \ INIT_ID(match), \ INIT_ID(max_length), \ + INIT_ID(max_threads), \ INIT_ID(maxdigits), \ INIT_ID(maxevents), \ INIT_ID(maxlen), \ INIT_ID(maxmem), \ + INIT_ID(maxsize), \ INIT_ID(maxsplit), \ INIT_ID(maxvalue), \ INIT_ID(memLevel), \ @@ -1089,13 +1911,16 @@ extern "C" { INIT_ID(metadata), \ INIT_ID(method), \ INIT_ID(microsecond), \ + INIT_ID(microseconds), \ INIT_ID(milliseconds), \ INIT_ID(minute), \ + INIT_ID(minutes), \ INIT_ID(mod), \ INIT_ID(mode), \ INIT_ID(module), \ INIT_ID(module_globals), \ INIT_ID(modules), \ + INIT_ID(modulo), \ INIT_ID(month), \ INIT_ID(mro), \ INIT_ID(msg), \ @@ -1108,6 +1933,7 @@ extern "C" { INIT_ID(name_from), \ INIT_ID(namespace_separator), \ INIT_ID(namespaces), \ + INIT_ID(native), \ INIT_ID(ndigits), \ INIT_ID(nested), \ INIT_ID(new_file_name), \ @@ -1134,6 +1960,7 @@ extern "C" { INIT_ID(only_keys), \ INIT_ID(oparg), \ INIT_ID(opcode), \ + INIT_ID(opcodes), \ INIT_ID(open), \ INIT_ID(opener), \ INIT_ID(operation), \ @@ -1141,11 +1968,14 @@ extern "C" { INIT_ID(options), \ INIT_ID(order), \ INIT_ID(origin), \ + INIT_ID(other), \ INIT_ID(out_fd), \ INIT_ID(outgoing), \ INIT_ID(outpath), \ INIT_ID(overlapped), \ INIT_ID(owner), \ + INIT_ID(pad), \ + INIT_ID(padded), \ INIT_ID(pages), \ INIT_ID(parameter), \ INIT_ID(parent), \ @@ -1158,20 +1988,28 @@ extern "C" { INIT_ID(person), \ INIT_ID(pi_factory), \ INIT_ID(pid), \ + INIT_ID(pidfd), \ + INIT_ID(pointer_bits), \ INIT_ID(policy), \ INIT_ID(pos), \ INIT_ID(pos1), \ INIT_ID(pos2), \ INIT_ID(posix), \ + INIT_ID(prec), \ + INIT_ID(preserve_exc), \ INIT_ID(print_file_and_line), \ INIT_ID(priority), \ INIT_ID(progress), \ + INIT_ID(progress_callback), \ INIT_ID(progress_routine), \ INIT_ID(proto), \ INIT_ID(protocol), \ INIT_ID(ps1), \ INIT_ID(ps2), \ + INIT_ID(qid), \ + INIT_ID(qualname), \ INIT_ID(query), \ + INIT_ID(queuetype), \ INIT_ID(quotetabs), \ INIT_ID(raw), \ INIT_ID(read), \ @@ -1183,24 +2021,33 @@ extern "C" { INIT_ID(readline), \ INIT_ID(readonly), \ INIT_ID(real), \ + INIT_ID(recursive), \ INIT_ID(reducer_override), \ INIT_ID(registry), \ INIT_ID(rel_tol), \ INIT_ID(release), \ INIT_ID(reload), \ + INIT_ID(repeat), \ INIT_ID(repl), \ INIT_ID(replace), \ + INIT_ID(repr), \ + INIT_ID(reqrefs), \ + INIT_ID(require_ready), \ INIT_ID(reserved), \ INIT_ID(reset), \ INIT_ID(resetids), \ + INIT_ID(restrict), \ INIT_ID(return), \ INIT_ID(reverse), \ INIT_ID(reversed), \ + INIT_ID(rounding), \ INIT_ID(salt), \ + INIT_ID(sample_interval_us), \ INIT_ID(sched_priority), \ INIT_ID(scheduler), \ INIT_ID(script), \ INIT_ID(second), \ + INIT_ID(seconds), \ INIT_ID(security_attributes), \ INIT_ID(seek), \ INIT_ID(seekable), \ @@ -1218,11 +2065,15 @@ extern "C" { INIT_ID(setsigmask), \ INIT_ID(setstate), \ INIT_ID(shape), \ + INIT_ID(shared), \ + INIT_ID(short), \ INIT_ID(show_cmd), \ INIT_ID(signed), \ + INIT_ID(signum), \ INIT_ID(size), \ INIT_ID(sizehint), \ INIT_ID(skip_file_prefixes), \ + INIT_ID(skip_non_matching_threads), \ INIT_ID(sleep), \ INIT_ID(sock), \ INIT_ID(sort), \ @@ -1231,9 +2082,12 @@ extern "C" { INIT_ID(spam), \ INIT_ID(src), \ INIT_ID(src_dir_fd), \ + INIT_ID(stack_frames), \ INIT_ID(stacklevel), \ INIT_ID(start), \ + INIT_ID(start_time_us), \ INIT_ID(statement), \ + INIT_ID(stats), \ INIT_ID(status), \ INIT_ID(stderr), \ INIT_ID(stdin), \ @@ -1251,8 +2105,10 @@ extern "C" { INIT_ID(symmetric_difference_update), \ INIT_ID(tabsize), \ INIT_ID(tag), \ + INIT_ID(take_bytes), \ INIT_ID(target), \ INIT_ID(target_is_directory), \ + INIT_ID(targetfd), \ INIT_ID(task), \ INIT_ID(tb_frame), \ INIT_ID(tb_lasti), \ @@ -1262,17 +2118,23 @@ extern "C" { INIT_ID(template), \ INIT_ID(term), \ INIT_ID(text), \ + INIT_ID(third), \ INIT_ID(threading), \ INIT_ID(throw), \ + INIT_ID(time), \ INIT_ID(timeout), \ INIT_ID(timer), \ INIT_ID(times), \ + INIT_ID(timespec), \ + INIT_ID(timestamp), \ + INIT_ID(timestamp_us), \ INIT_ID(timetuple), \ INIT_ID(timeunit), \ INIT_ID(top), \ INIT_ID(traceback), \ INIT_ID(trailers), \ INIT_ID(translate), \ + INIT_ID(traps), \ INIT_ID(true), \ INIT_ID(truncate), \ INIT_ID(twice), \ @@ -1283,10 +2145,13 @@ extern "C" { INIT_ID(tzinfo), \ INIT_ID(tzname), \ INIT_ID(uid), \ + INIT_ID(unboundop), \ INIT_ID(unlink), \ INIT_ID(unraisablehook), \ + INIT_ID(updates), \ INIT_ID(uri), \ INIT_ID(usedforsecurity), \ + INIT_ID(utcoffset), \ INIT_ID(value), \ INIT_ID(values), \ INIT_ID(version), \ @@ -1298,9 +2163,11 @@ extern "C" { INIT_ID(wbits), \ INIT_ID(week), \ INIT_ID(weekday), \ + INIT_ID(weeks), \ INIT_ID(which), \ INIT_ID(who), \ INIT_ID(withdata), \ + INIT_ID(wrapcol), \ INIT_ID(writable), \ INIT_ID(write), \ INIT_ID(write_through), \ diff --git a/Include/internal/pycore_runtime_structs.h b/Include/internal/pycore_runtime_structs.h index 12164c7fdd9425a..145e66de9984ca7 100644 --- a/Include/internal/pycore_runtime_structs.h +++ b/Include/internal/pycore_runtime_structs.h @@ -31,20 +31,10 @@ struct _pymem_allocators { debug_alloc_api_t obj; } debug; int is_debug_enabled; + int use_hugepages; PyObjectArenaAllocator obj_arena; }; -enum _py_float_format_type { - _py_float_format_unknown, - _py_float_format_ieee_big_endian, - _py_float_format_ieee_little_endian, -}; - -struct _Py_float_runtime_state { - enum _py_float_format_type float_format; - enum _py_float_format_type double_format; -}; - struct pyhash_runtime_state { struct { #ifndef MS_WINDOWS @@ -77,9 +67,7 @@ struct _fileutils_state { struct _parser_runtime_state { #ifdef Py_DEBUG long memo_statistics[_PYPEGEN_NSTATISTICS]; -#ifdef Py_GIL_DISABLED PyMutex mutex; -#endif #else int _not_used; #endif @@ -106,7 +94,7 @@ struct _Py_cached_objects { }; // These would be in pycore_long.h if it weren't for an include cycle. -#define _PY_NSMALLPOSINTS 257 +#define _PY_NSMALLPOSINTS 1025 #define _PY_NSMALLNEGINTS 5 #include "pycore_global_strings.h" // struct _Py_global_strings @@ -170,10 +158,18 @@ struct pyruntimestate { /* Is Python preinitialized? Set to 1 by Py_PreInitialize() */ int preinitialized; - /* Is Python core initialized? Set to 1 by _Py_InitializeCore() */ + /* Is Python core initialized? Set to 1 by _Py_InitializeCore(). + + Use _PyRuntimeState_GetCoreInitialized() and + _PyRuntimeState_SetCoreInitialized() to access it, + don't access it directly. */ int core_initialized; - /* Is Python fully initialized? Set to 1 by Py_Initialize() */ + /* Is Python fully initialized? Set to 1 by Py_Initialize(). + + Use _PyRuntimeState_GetInitialized() and + _PyRuntimeState_SetInitialized() to access it, + don't access it directly. */ int initialized; /* Set by Py_FinalizeEx(). Only reset to NULL if Py_Initialize() @@ -271,15 +267,18 @@ struct pyruntimestate { } audit_hooks; struct _py_object_runtime_state object_state; - struct _Py_float_runtime_state float_state; struct _Py_unicode_runtime_state unicode_state; struct _types_runtime_state types; struct _Py_time_runtime_state time; #if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) - // Used in "Python/emscripten_trampoline.c" to choose between type - // reflection trampoline and EM_JS trampoline. - int (*emscripten_count_args_function)(PyCFunctionWithKeywords func); + // Used in "Python/emscripten_trampoline.c" to choose between wasm-gc + // trampoline and JavaScript trampoline. + PyObject* (*emscripten_trampoline)(int* success, + PyCFunctionWithKeywords func, + PyObject* self, + PyObject* args, + PyObject* kw); #endif /* All the objects that are shared by the runtime's interpreters. */ diff --git a/Include/internal/pycore_semaphore.h b/Include/internal/pycore_semaphore.h index 269538384606ce1..66b4939dcacf051 100644 --- a/Include/internal/pycore_semaphore.h +++ b/Include/internal/pycore_semaphore.h @@ -46,10 +46,8 @@ typedef struct _PySemaphore { } _PySemaphore; // Puts the current thread to sleep until _PySemaphore_Wakeup() is called. -// If `detach` is true, then the thread will detach/release the GIL while -// sleeping. PyAPI_FUNC(int) -_PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout_ns, int detach); +_PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout_ns); // Wakes up a single thread waiting on sema. Note that _PySemaphore_Wakeup() // can be called before _PySemaphore_Wait(). diff --git a/Include/internal/pycore_setobject.h b/Include/internal/pycore_setobject.h index 24d0135ed1aeca8..92d1a15177f79eb 100644 --- a/Include/internal/pycore_setobject.h +++ b/Include/internal/pycore_setobject.h @@ -35,6 +35,9 @@ extern void _PySet_ClearInternal(PySetObject *so); PyAPI_FUNC(int) _PySet_AddTakeRef(PySetObject *so, PyObject *key); +PyObject * +_PySet_Freeze(PyObject *set); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_sliceobject.h b/Include/internal/pycore_sliceobject.h index ba8b1f1cb27dee3..b6c821764886c36 100644 --- a/Include/internal/pycore_sliceobject.h +++ b/Include/internal/pycore_sliceobject.h @@ -12,7 +12,7 @@ extern "C" { /* runtime lifecycle */ PyAPI_FUNC(PyObject *) -_PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop); +_PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop, PyObject *step); #ifdef __cplusplus } diff --git a/Include/internal/pycore_slots.h b/Include/internal/pycore_slots.h new file mode 100644 index 000000000000000..960a632aea17eea --- /dev/null +++ b/Include/internal/pycore_slots.h @@ -0,0 +1,138 @@ +#ifndef _Py_PYCORE_SLOTS_H +#define _Py_PYCORE_SLOTS_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include + +/* Slot data type */ +typedef enum _PySlot_DTYPE { + _PySlot_DTYPE_VOID, + _PySlot_DTYPE_FUNC, + _PySlot_DTYPE_PTR, + _PySlot_DTYPE_SIZE, + _PySlot_DTYPE_INT64, + _PySlot_DTYPE_UINT64, +}_PySlot_DTYPE; + +/* Slot kind, used to identify: + * - the thing the slot initializes (type/module/special) + * - the struct type (PySlot/PyType_Slot/PyModuleDef_Slot) + */ +typedef enum _PySlot_KIND { + _PySlot_KIND_TYPE, + _PySlot_KIND_MOD, + _PySlot_KIND_COMPAT, + _PySlot_KIND_SLOT, +} _PySlot_KIND; + +typedef enum _PySlot_PROBLEM_HANDLING { + _PySlot_PROBLEM_ALLOW, + _PySlot_PROBLEM_DEPRECATED, + _PySlot_PROBLEM_REJECT, +} _PySlot_PROBLEM_HANDLING; + +PyAPI_DATA(const char *const) _PySlot_names[]; + +#define _PySlot_MAX_NESTING 5 + +/* State for one nesting level of a slots iterator */ +typedef struct _PySlotIterator_state { + union { + // tagged by slot_struct_kind: + const PySlot *slot; // with _PySlot_KIND_SLOT + const PyType_Slot *tp_slot; // with _PySlot_KIND_TYPE + const PyModuleDef_Slot *mod_slot; // with _PySlot_KIND_MOD + const void *any_slot; + }; + _PySlot_KIND slot_struct_kind; +} _PySlotIterator_state; + +#define _PySlot_SEEN_ENTRY_BITS (8 * sizeof(unsigned int)) + +/* State for a slots iterator */ +typedef struct { + _PySlotIterator_state *state; + _PySlotIterator_state states[_PySlot_MAX_NESTING]; + unsigned int seen[_Py_slot_COUNT / _PySlot_SEEN_ENTRY_BITS + 1]; + _PySlot_KIND kind; + uint8_t recursion_level; + bool is_at_end :1; + bool is_first_run :1; + + // Name of the object (type/module) being defined, NULL if unknown. + // Must be set by the callers as soon as it's known. + const char *name; + + /* Output information: */ + + // The slot. Always a copy; may be modified by caller of the iterator. + PySlot current; + +} _PySlotIterator; + +/* Initialize an iterator using a PySlot array */ +PyAPI_FUNC(void) +_PySlotIterator_Init(_PySlotIterator *it, const PySlot *slots, + _PySlot_KIND result_kind); + +/* Initialize an iterator using a legacy slot array */ +PyAPI_FUNC(void) +_PySlotIterator_InitLegacy(_PySlotIterator *it, const void *slots, + _PySlot_KIND kind); + +/* Reset a *successfully exhausted* iterator to the beginning. + * The *slots* must be the same as for the previous + * `_PySlotIterator_InitWithKind` call. + * (Unlike creating a new iterator, we can skip some validation after Rewind.) + */ +PyAPI_FUNC(void) _PySlotIterator_Rewind(_PySlotIterator *it, const void *slots); + +/* Iteration function. + * + * Return false at the end (when successfully exhausted). + * Otherwise (even on error), fill output information in `it` and return true. + * + * On error, set an exception and set `it->current.sl_id` to `Py_slot_invalid`. + */ +PyAPI_FUNC(bool) _PySlotIterator_Next(_PySlotIterator *it); + +/* Return 1 if given slot was "seen" by an earlier _PySlotIterator_Next call. + * (This state is not reset by rewinding.) + */ +PyAPI_FUNC(bool) _PySlotIterator_SawSlot(_PySlotIterator *, int); + +static inline const char * +_PySlot_GetName(uint16_t id) +{ + if (id >= _Py_slot_COUNT) { + return ""; + } + if (id == Py_slot_invalid) { + return "Py_slot_invalid"; + } + return _PySlot_names[id]; +} + +static inline void +_PySlot_err_bad_slot(char *kind, uint16_t id) +{ + if (id < _Py_slot_COUNT) { + PyErr_Format(PyExc_SystemError, "invalid %s slot %d (%s)", + kind, (unsigned int)id, _PySlot_names[id]); + } + else if (id == Py_slot_invalid) { + PyErr_Format(PyExc_SystemError, "invalid slot (Py_slot_invalid, %u)", + (unsigned int)id); + } + else { + PyErr_Format(PyExc_SystemError, "unknown %s slot ID %u", + kind, (unsigned int)id); + } +} + +#include "internal/pycore_slots_generated.h" + +#endif // _Py_PYCORE_SLOTS_H diff --git a/Include/internal/pycore_slots_generated.h b/Include/internal/pycore_slots_generated.h new file mode 100644 index 000000000000000..73a77070038cef4 --- /dev/null +++ b/Include/internal/pycore_slots_generated.h @@ -0,0 +1,958 @@ +/* Generated by Tools/build/generate_slots.py */ + +#ifndef _PY_HAVE_INTERNAL_SLOTS_GENERATED_H +#define _PY_HAVE_INTERNAL_SLOTS_GENERATED_H + +static inline uint16_t +_PySlot_resolve_type_slot(uint16_t slot_id) +{ + switch (slot_id) { + case 1: + return Py_bf_getbuffer; + case 2: + return Py_bf_releasebuffer; + case 3: + return Py_mp_ass_subscript; + case 4: + return Py_mp_length; + case Py_slot_end: + case Py_mp_subscript: + case Py_nb_absolute: + case Py_nb_add: + case Py_nb_and: + case Py_nb_bool: + case Py_nb_divmod: + case Py_nb_float: + case Py_nb_floor_divide: + case Py_nb_index: + case Py_nb_inplace_add: + case Py_nb_inplace_and: + case Py_nb_inplace_floor_divide: + case Py_nb_inplace_lshift: + case Py_nb_inplace_multiply: + case Py_nb_inplace_or: + case Py_nb_inplace_power: + case Py_nb_inplace_remainder: + case Py_nb_inplace_rshift: + case Py_nb_inplace_subtract: + case Py_nb_inplace_true_divide: + case Py_nb_inplace_xor: + case Py_nb_int: + case Py_nb_invert: + case Py_nb_lshift: + case Py_nb_multiply: + case Py_nb_negative: + case Py_nb_or: + case Py_nb_positive: + case Py_nb_power: + case Py_nb_remainder: + case Py_nb_rshift: + case Py_nb_subtract: + case Py_nb_true_divide: + case Py_nb_xor: + case Py_sq_ass_item: + case Py_sq_concat: + case Py_sq_contains: + case Py_sq_inplace_concat: + case Py_sq_inplace_repeat: + case Py_sq_item: + case Py_sq_length: + case Py_sq_repeat: + case Py_tp_alloc: + case Py_tp_base: + case Py_tp_bases: + case Py_tp_call: + case Py_tp_clear: + case Py_tp_dealloc: + case Py_tp_del: + case Py_tp_descr_get: + case Py_tp_descr_set: + case Py_tp_doc: + case Py_tp_getattr: + case Py_tp_getattro: + case Py_tp_hash: + case Py_tp_init: + case Py_tp_is_gc: + case Py_tp_iter: + case Py_tp_iternext: + case Py_tp_methods: + case Py_tp_new: + case Py_tp_repr: + case Py_tp_richcompare: + case Py_tp_setattr: + case Py_tp_setattro: + case Py_tp_str: + case Py_tp_traverse: + case Py_tp_members: + case Py_tp_getset: + case Py_tp_free: + case Py_nb_matrix_multiply: + case Py_nb_inplace_matrix_multiply: + case Py_am_await: + case Py_am_aiter: + case Py_am_anext: + case Py_tp_finalize: + case Py_am_send: + case Py_tp_vectorcall: + case Py_tp_token: + case Py_bf_getbuffer: + case Py_bf_releasebuffer: + case Py_mp_ass_subscript: + case Py_mp_length: + case Py_slot_subslots: + case Py_tp_slots: + case Py_tp_name: + case Py_tp_basicsize: + case Py_tp_extra_basicsize: + case Py_tp_itemsize: + case Py_tp_flags: + case Py_tp_metaclass: + case Py_tp_module: + return slot_id; + default: + return Py_slot_invalid; + } +} + +static inline uint16_t +_PySlot_resolve_mod_slot(uint16_t slot_id) +{ + switch (slot_id) { + case 1: + return Py_mod_create; + case 2: + return Py_mod_exec; + case 3: + return Py_mod_multiple_interpreters; + case 4: + return Py_mod_gil; + case Py_slot_end: + case Py_mod_create: + case Py_mod_exec: + case Py_mod_multiple_interpreters: + case Py_mod_gil: + case Py_slot_subslots: + case Py_mod_slots: + case Py_mod_name: + case Py_mod_doc: + case Py_mod_state_size: + case Py_mod_methods: + case Py_mod_state_traverse: + case Py_mod_state_clear: + case Py_mod_state_free: + case Py_mod_abi: + case Py_mod_token: + return slot_id; + default: + return Py_slot_invalid; + } +} + +static inline void* +_PySlot_type_getslot(PyTypeObject *tp, uint16_t slot_id) +{ + switch (slot_id) { + case Py_mp_subscript: + if (!(tp->tp_as_mapping)) return NULL; + return (void*)tp->tp_as_mapping->mp_subscript; + case Py_nb_absolute: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_absolute; + case Py_nb_add: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_add; + case Py_nb_and: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_and; + case Py_nb_bool: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_bool; + case Py_nb_divmod: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_divmod; + case Py_nb_float: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_float; + case Py_nb_floor_divide: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_floor_divide; + case Py_nb_index: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_index; + case Py_nb_inplace_add: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_add; + case Py_nb_inplace_and: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_and; + case Py_nb_inplace_floor_divide: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_floor_divide; + case Py_nb_inplace_lshift: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_lshift; + case Py_nb_inplace_multiply: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_multiply; + case Py_nb_inplace_or: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_or; + case Py_nb_inplace_power: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_power; + case Py_nb_inplace_remainder: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_remainder; + case Py_nb_inplace_rshift: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_rshift; + case Py_nb_inplace_subtract: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_subtract; + case Py_nb_inplace_true_divide: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_true_divide; + case Py_nb_inplace_xor: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_xor; + case Py_nb_int: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_int; + case Py_nb_invert: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_invert; + case Py_nb_lshift: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_lshift; + case Py_nb_multiply: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_multiply; + case Py_nb_negative: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_negative; + case Py_nb_or: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_or; + case Py_nb_positive: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_positive; + case Py_nb_power: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_power; + case Py_nb_remainder: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_remainder; + case Py_nb_rshift: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_rshift; + case Py_nb_subtract: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_subtract; + case Py_nb_true_divide: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_true_divide; + case Py_nb_xor: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_xor; + case Py_sq_ass_item: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_ass_item; + case Py_sq_concat: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_concat; + case Py_sq_contains: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_contains; + case Py_sq_inplace_concat: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_inplace_concat; + case Py_sq_inplace_repeat: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_inplace_repeat; + case Py_sq_item: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_item; + case Py_sq_length: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_length; + case Py_sq_repeat: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_repeat; + case Py_tp_alloc: + return (void*)tp->tp_alloc; + case Py_tp_base: + return (void*)tp->tp_base; + case Py_tp_bases: + return (void*)tp->tp_bases; + case Py_tp_call: + return (void*)tp->tp_call; + case Py_tp_clear: + return (void*)tp->tp_clear; + case Py_tp_dealloc: + return (void*)tp->tp_dealloc; + case Py_tp_del: + return (void*)tp->tp_del; + case Py_tp_descr_get: + return (void*)tp->tp_descr_get; + case Py_tp_descr_set: + return (void*)tp->tp_descr_set; + case Py_tp_doc: + return (void*)tp->tp_doc; + case Py_tp_getattr: + return (void*)tp->tp_getattr; + case Py_tp_getattro: + return (void*)tp->tp_getattro; + case Py_tp_hash: + return (void*)tp->tp_hash; + case Py_tp_init: + return (void*)tp->tp_init; + case Py_tp_is_gc: + return (void*)tp->tp_is_gc; + case Py_tp_iter: + return (void*)tp->tp_iter; + case Py_tp_iternext: + return (void*)tp->tp_iternext; + case Py_tp_methods: + return (void*)tp->tp_methods; + case Py_tp_new: + return (void*)tp->tp_new; + case Py_tp_repr: + return (void*)tp->tp_repr; + case Py_tp_richcompare: + return (void*)tp->tp_richcompare; + case Py_tp_setattr: + return (void*)tp->tp_setattr; + case Py_tp_setattro: + return (void*)tp->tp_setattro; + case Py_tp_str: + return (void*)tp->tp_str; + case Py_tp_traverse: + return (void*)tp->tp_traverse; + case Py_tp_members: + return (void*)tp->tp_members; + case Py_tp_getset: + return (void*)tp->tp_getset; + case Py_tp_free: + return (void*)tp->tp_free; + case Py_nb_matrix_multiply: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_matrix_multiply; + case Py_nb_inplace_matrix_multiply: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_matrix_multiply; + case Py_am_await: + if (!(tp->tp_as_async)) return NULL; + return (void*)tp->tp_as_async->am_await; + case Py_am_aiter: + if (!(tp->tp_as_async)) return NULL; + return (void*)tp->tp_as_async->am_aiter; + case Py_am_anext: + if (!(tp->tp_as_async)) return NULL; + return (void*)tp->tp_as_async->am_anext; + case Py_tp_finalize: + return (void*)tp->tp_finalize; + case Py_am_send: + if (!(tp->tp_as_async)) return NULL; + return (void*)tp->tp_as_async->am_send; + case Py_tp_vectorcall: + return (void*)tp->tp_vectorcall; + case Py_tp_token: + if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) return NULL; + return (void*)((PyHeapTypeObject*)tp)->ht_token; + case Py_bf_getbuffer: + if (!(tp->tp_as_buffer)) return NULL; + return (void*)tp->tp_as_buffer->bf_getbuffer; + case Py_bf_releasebuffer: + if (!(tp->tp_as_buffer)) return NULL; + return (void*)tp->tp_as_buffer->bf_releasebuffer; + case Py_mp_ass_subscript: + if (!(tp->tp_as_mapping)) return NULL; + return (void*)tp->tp_as_mapping->mp_ass_subscript; + case Py_mp_length: + if (!(tp->tp_as_mapping)) return NULL; + return (void*)tp->tp_as_mapping->mp_length; + } + _PySlot_err_bad_slot("PyType_GetSlot", slot_id); + return NULL; +} + +static inline void +_PySlot_heaptype_apply_field_slot(PyHeapTypeObject *ht, PySlot slot) +{ + switch (slot.sl_id) { + case Py_mp_subscript: + ht->as_mapping.mp_subscript = (binaryfunc)slot.sl_func; + break; + case Py_nb_absolute: + ht->as_number.nb_absolute = (unaryfunc)slot.sl_func; + break; + case Py_nb_add: + ht->as_number.nb_add = (binaryfunc)slot.sl_func; + break; + case Py_nb_and: + ht->as_number.nb_and = (binaryfunc)slot.sl_func; + break; + case Py_nb_bool: + ht->as_number.nb_bool = (inquiry)slot.sl_func; + break; + case Py_nb_divmod: + ht->as_number.nb_divmod = (binaryfunc)slot.sl_func; + break; + case Py_nb_float: + ht->as_number.nb_float = (unaryfunc)slot.sl_func; + break; + case Py_nb_floor_divide: + ht->as_number.nb_floor_divide = (binaryfunc)slot.sl_func; + break; + case Py_nb_index: + ht->as_number.nb_index = (unaryfunc)slot.sl_func; + break; + case Py_nb_inplace_add: + ht->as_number.nb_inplace_add = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_and: + ht->as_number.nb_inplace_and = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_floor_divide: + ht->as_number.nb_inplace_floor_divide = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_lshift: + ht->as_number.nb_inplace_lshift = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_multiply: + ht->as_number.nb_inplace_multiply = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_or: + ht->as_number.nb_inplace_or = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_power: + ht->as_number.nb_inplace_power = (ternaryfunc)slot.sl_func; + break; + case Py_nb_inplace_remainder: + ht->as_number.nb_inplace_remainder = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_rshift: + ht->as_number.nb_inplace_rshift = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_subtract: + ht->as_number.nb_inplace_subtract = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_true_divide: + ht->as_number.nb_inplace_true_divide = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_xor: + ht->as_number.nb_inplace_xor = (binaryfunc)slot.sl_func; + break; + case Py_nb_int: + ht->as_number.nb_int = (unaryfunc)slot.sl_func; + break; + case Py_nb_invert: + ht->as_number.nb_invert = (unaryfunc)slot.sl_func; + break; + case Py_nb_lshift: + ht->as_number.nb_lshift = (binaryfunc)slot.sl_func; + break; + case Py_nb_multiply: + ht->as_number.nb_multiply = (binaryfunc)slot.sl_func; + break; + case Py_nb_negative: + ht->as_number.nb_negative = (unaryfunc)slot.sl_func; + break; + case Py_nb_or: + ht->as_number.nb_or = (binaryfunc)slot.sl_func; + break; + case Py_nb_positive: + ht->as_number.nb_positive = (unaryfunc)slot.sl_func; + break; + case Py_nb_power: + ht->as_number.nb_power = (ternaryfunc)slot.sl_func; + break; + case Py_nb_remainder: + ht->as_number.nb_remainder = (binaryfunc)slot.sl_func; + break; + case Py_nb_rshift: + ht->as_number.nb_rshift = (binaryfunc)slot.sl_func; + break; + case Py_nb_subtract: + ht->as_number.nb_subtract = (binaryfunc)slot.sl_func; + break; + case Py_nb_true_divide: + ht->as_number.nb_true_divide = (binaryfunc)slot.sl_func; + break; + case Py_nb_xor: + ht->as_number.nb_xor = (binaryfunc)slot.sl_func; + break; + case Py_sq_ass_item: + ht->as_sequence.sq_ass_item = (ssizeobjargproc)slot.sl_func; + break; + case Py_sq_concat: + ht->as_sequence.sq_concat = (binaryfunc)slot.sl_func; + break; + case Py_sq_contains: + ht->as_sequence.sq_contains = (objobjproc)slot.sl_func; + break; + case Py_sq_inplace_concat: + ht->as_sequence.sq_inplace_concat = (binaryfunc)slot.sl_func; + break; + case Py_sq_inplace_repeat: + ht->as_sequence.sq_inplace_repeat = (ssizeargfunc)slot.sl_func; + break; + case Py_sq_item: + ht->as_sequence.sq_item = (ssizeargfunc)slot.sl_func; + break; + case Py_sq_length: + ht->as_sequence.sq_length = (lenfunc)slot.sl_func; + break; + case Py_sq_repeat: + ht->as_sequence.sq_repeat = (ssizeargfunc)slot.sl_func; + break; + case Py_tp_alloc: + ht->ht_type.tp_alloc = (allocfunc)slot.sl_func; + break; + case Py_tp_base: + ht->ht_type.tp_base = slot.sl_ptr; + break; + case Py_tp_bases: + ht->ht_type.tp_bases = slot.sl_ptr; + break; + case Py_tp_call: + ht->ht_type.tp_call = (ternaryfunc)slot.sl_func; + break; + case Py_tp_clear: + ht->ht_type.tp_clear = (inquiry)slot.sl_func; + break; + case Py_tp_dealloc: + ht->ht_type.tp_dealloc = (destructor)slot.sl_func; + break; + case Py_tp_del: + ht->ht_type.tp_del = (destructor)slot.sl_func; + break; + case Py_tp_descr_get: + ht->ht_type.tp_descr_get = (descrgetfunc)slot.sl_func; + break; + case Py_tp_descr_set: + ht->ht_type.tp_descr_set = (descrsetfunc)slot.sl_func; + break; + case Py_tp_doc: + ht->ht_type.tp_doc = slot.sl_ptr; + break; + case Py_tp_getattr: + ht->ht_type.tp_getattr = (getattrfunc)slot.sl_func; + break; + case Py_tp_getattro: + ht->ht_type.tp_getattro = (getattrofunc)slot.sl_func; + break; + case Py_tp_hash: + ht->ht_type.tp_hash = (hashfunc)slot.sl_func; + break; + case Py_tp_init: + ht->ht_type.tp_init = (initproc)slot.sl_func; + break; + case Py_tp_is_gc: + ht->ht_type.tp_is_gc = (inquiry)slot.sl_func; + break; + case Py_tp_iter: + ht->ht_type.tp_iter = (getiterfunc)slot.sl_func; + break; + case Py_tp_iternext: + ht->ht_type.tp_iternext = (iternextfunc)slot.sl_func; + break; + case Py_tp_methods: + ht->ht_type.tp_methods = slot.sl_ptr; + break; + case Py_tp_new: + ht->ht_type.tp_new = (newfunc)slot.sl_func; + break; + case Py_tp_repr: + ht->ht_type.tp_repr = (reprfunc)slot.sl_func; + break; + case Py_tp_richcompare: + ht->ht_type.tp_richcompare = (richcmpfunc)slot.sl_func; + break; + case Py_tp_setattr: + ht->ht_type.tp_setattr = (setattrfunc)slot.sl_func; + break; + case Py_tp_setattro: + ht->ht_type.tp_setattro = (setattrofunc)slot.sl_func; + break; + case Py_tp_str: + ht->ht_type.tp_str = (reprfunc)slot.sl_func; + break; + case Py_tp_traverse: + ht->ht_type.tp_traverse = (traverseproc)slot.sl_func; + break; + case Py_tp_members: + ht->ht_type.tp_members = slot.sl_ptr; + break; + case Py_tp_getset: + ht->ht_type.tp_getset = slot.sl_ptr; + break; + case Py_tp_free: + ht->ht_type.tp_free = (freefunc)slot.sl_func; + break; + case Py_nb_matrix_multiply: + ht->as_number.nb_matrix_multiply = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_matrix_multiply: + ht->as_number.nb_inplace_matrix_multiply = (binaryfunc)slot.sl_func; + break; + case Py_am_await: + ht->as_async.am_await = (unaryfunc)slot.sl_func; + break; + case Py_am_aiter: + ht->as_async.am_aiter = (unaryfunc)slot.sl_func; + break; + case Py_am_anext: + ht->as_async.am_anext = (unaryfunc)slot.sl_func; + break; + case Py_tp_finalize: + ht->ht_type.tp_finalize = (destructor)slot.sl_func; + break; + case Py_am_send: + ht->as_async.am_send = (sendfunc)slot.sl_func; + break; + case Py_tp_vectorcall: + ht->ht_type.tp_vectorcall = (vectorcallfunc)slot.sl_func; + break; + case Py_bf_getbuffer: + ht->as_buffer.bf_getbuffer = (getbufferproc)slot.sl_func; + break; + case Py_bf_releasebuffer: + ht->as_buffer.bf_releasebuffer = (releasebufferproc)slot.sl_func; + break; + case Py_mp_ass_subscript: + ht->as_mapping.mp_ass_subscript = (objobjargproc)slot.sl_func; + break; + case Py_mp_length: + ht->as_mapping.mp_length = (lenfunc)slot.sl_func; + break; + } +} + +static inline _PySlot_DTYPE +_PySlot_get_dtype(uint16_t slot_id) +{ + switch (slot_id) { + case Py_slot_end: return _PySlot_DTYPE_VOID; + case Py_mp_subscript: return _PySlot_DTYPE_FUNC; + case Py_nb_absolute: return _PySlot_DTYPE_FUNC; + case Py_nb_add: return _PySlot_DTYPE_FUNC; + case Py_nb_and: return _PySlot_DTYPE_FUNC; + case Py_nb_bool: return _PySlot_DTYPE_FUNC; + case Py_nb_divmod: return _PySlot_DTYPE_FUNC; + case Py_nb_float: return _PySlot_DTYPE_FUNC; + case Py_nb_floor_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_index: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_add: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_and: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_floor_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_lshift: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_multiply: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_or: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_power: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_remainder: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_rshift: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_subtract: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_true_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_xor: return _PySlot_DTYPE_FUNC; + case Py_nb_int: return _PySlot_DTYPE_FUNC; + case Py_nb_invert: return _PySlot_DTYPE_FUNC; + case Py_nb_lshift: return _PySlot_DTYPE_FUNC; + case Py_nb_multiply: return _PySlot_DTYPE_FUNC; + case Py_nb_negative: return _PySlot_DTYPE_FUNC; + case Py_nb_or: return _PySlot_DTYPE_FUNC; + case Py_nb_positive: return _PySlot_DTYPE_FUNC; + case Py_nb_power: return _PySlot_DTYPE_FUNC; + case Py_nb_remainder: return _PySlot_DTYPE_FUNC; + case Py_nb_rshift: return _PySlot_DTYPE_FUNC; + case Py_nb_subtract: return _PySlot_DTYPE_FUNC; + case Py_nb_true_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_xor: return _PySlot_DTYPE_FUNC; + case Py_sq_ass_item: return _PySlot_DTYPE_FUNC; + case Py_sq_concat: return _PySlot_DTYPE_FUNC; + case Py_sq_contains: return _PySlot_DTYPE_FUNC; + case Py_sq_inplace_concat: return _PySlot_DTYPE_FUNC; + case Py_sq_inplace_repeat: return _PySlot_DTYPE_FUNC; + case Py_sq_item: return _PySlot_DTYPE_FUNC; + case Py_sq_length: return _PySlot_DTYPE_FUNC; + case Py_sq_repeat: return _PySlot_DTYPE_FUNC; + case Py_tp_alloc: return _PySlot_DTYPE_FUNC; + case Py_tp_base: return _PySlot_DTYPE_PTR; + case Py_tp_bases: return _PySlot_DTYPE_PTR; + case Py_tp_call: return _PySlot_DTYPE_FUNC; + case Py_tp_clear: return _PySlot_DTYPE_FUNC; + case Py_tp_dealloc: return _PySlot_DTYPE_FUNC; + case Py_tp_del: return _PySlot_DTYPE_FUNC; + case Py_tp_descr_get: return _PySlot_DTYPE_FUNC; + case Py_tp_descr_set: return _PySlot_DTYPE_FUNC; + case Py_tp_doc: return _PySlot_DTYPE_PTR; + case Py_tp_getattr: return _PySlot_DTYPE_FUNC; + case Py_tp_getattro: return _PySlot_DTYPE_FUNC; + case Py_tp_hash: return _PySlot_DTYPE_FUNC; + case Py_tp_init: return _PySlot_DTYPE_FUNC; + case Py_tp_is_gc: return _PySlot_DTYPE_FUNC; + case Py_tp_iter: return _PySlot_DTYPE_FUNC; + case Py_tp_iternext: return _PySlot_DTYPE_FUNC; + case Py_tp_methods: return _PySlot_DTYPE_PTR; + case Py_tp_new: return _PySlot_DTYPE_FUNC; + case Py_tp_repr: return _PySlot_DTYPE_FUNC; + case Py_tp_richcompare: return _PySlot_DTYPE_FUNC; + case Py_tp_setattr: return _PySlot_DTYPE_FUNC; + case Py_tp_setattro: return _PySlot_DTYPE_FUNC; + case Py_tp_str: return _PySlot_DTYPE_FUNC; + case Py_tp_traverse: return _PySlot_DTYPE_FUNC; + case Py_tp_members: return _PySlot_DTYPE_PTR; + case Py_tp_getset: return _PySlot_DTYPE_PTR; + case Py_tp_free: return _PySlot_DTYPE_FUNC; + case Py_nb_matrix_multiply: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_matrix_multiply: return _PySlot_DTYPE_FUNC; + case Py_am_await: return _PySlot_DTYPE_FUNC; + case Py_am_aiter: return _PySlot_DTYPE_FUNC; + case Py_am_anext: return _PySlot_DTYPE_FUNC; + case Py_tp_finalize: return _PySlot_DTYPE_FUNC; + case Py_am_send: return _PySlot_DTYPE_FUNC; + case Py_tp_vectorcall: return _PySlot_DTYPE_FUNC; + case Py_tp_token: return _PySlot_DTYPE_PTR; + case Py_mod_create: return _PySlot_DTYPE_FUNC; + case Py_mod_exec: return _PySlot_DTYPE_FUNC; + case Py_mod_multiple_interpreters: return _PySlot_DTYPE_UINT64; + case Py_mod_gil: return _PySlot_DTYPE_UINT64; + case Py_bf_getbuffer: return _PySlot_DTYPE_FUNC; + case Py_bf_releasebuffer: return _PySlot_DTYPE_FUNC; + case Py_mp_ass_subscript: return _PySlot_DTYPE_FUNC; + case Py_mp_length: return _PySlot_DTYPE_FUNC; + case Py_slot_subslots: return _PySlot_DTYPE_PTR; + case Py_tp_slots: return _PySlot_DTYPE_PTR; + case Py_mod_slots: return _PySlot_DTYPE_PTR; + case Py_tp_name: return _PySlot_DTYPE_PTR; + case Py_tp_basicsize: return _PySlot_DTYPE_SIZE; + case Py_tp_extra_basicsize: return _PySlot_DTYPE_SIZE; + case Py_tp_itemsize: return _PySlot_DTYPE_SIZE; + case Py_tp_flags: return _PySlot_DTYPE_UINT64; + case Py_mod_name: return _PySlot_DTYPE_PTR; + case Py_mod_doc: return _PySlot_DTYPE_PTR; + case Py_mod_state_size: return _PySlot_DTYPE_SIZE; + case Py_mod_methods: return _PySlot_DTYPE_PTR; + case Py_mod_state_traverse: return _PySlot_DTYPE_FUNC; + case Py_mod_state_clear: return _PySlot_DTYPE_FUNC; + case Py_mod_state_free: return _PySlot_DTYPE_FUNC; + case Py_tp_metaclass: return _PySlot_DTYPE_PTR; + case Py_tp_module: return _PySlot_DTYPE_PTR; + case Py_mod_abi: return _PySlot_DTYPE_PTR; + case Py_mod_token: return _PySlot_DTYPE_PTR; + default: return _PySlot_DTYPE_VOID; + } +} + +static inline _PySlot_PROBLEM_HANDLING +_PySlot_get_duplicate_handling(uint16_t slot_id) +{ + switch (slot_id) { + case Py_mp_subscript: + case Py_nb_absolute: + case Py_nb_add: + case Py_nb_and: + case Py_nb_bool: + case Py_nb_divmod: + case Py_nb_float: + case Py_nb_floor_divide: + case Py_nb_index: + case Py_nb_inplace_add: + case Py_nb_inplace_and: + case Py_nb_inplace_floor_divide: + case Py_nb_inplace_lshift: + case Py_nb_inplace_multiply: + case Py_nb_inplace_or: + case Py_nb_inplace_power: + case Py_nb_inplace_remainder: + case Py_nb_inplace_rshift: + case Py_nb_inplace_subtract: + case Py_nb_inplace_true_divide: + case Py_nb_inplace_xor: + case Py_nb_int: + case Py_nb_invert: + case Py_nb_lshift: + case Py_nb_multiply: + case Py_nb_negative: + case Py_nb_or: + case Py_nb_positive: + case Py_nb_power: + case Py_nb_remainder: + case Py_nb_rshift: + case Py_nb_subtract: + case Py_nb_true_divide: + case Py_nb_xor: + case Py_sq_ass_item: + case Py_sq_concat: + case Py_sq_contains: + case Py_sq_inplace_concat: + case Py_sq_inplace_repeat: + case Py_sq_item: + case Py_sq_length: + case Py_sq_repeat: + case Py_tp_alloc: + case Py_tp_base: + case Py_tp_bases: + case Py_tp_call: + case Py_tp_clear: + case Py_tp_dealloc: + case Py_tp_del: + case Py_tp_descr_get: + case Py_tp_descr_set: + case Py_tp_getattr: + case Py_tp_getattro: + case Py_tp_hash: + case Py_tp_init: + case Py_tp_is_gc: + case Py_tp_iter: + case Py_tp_iternext: + case Py_tp_methods: + case Py_tp_new: + case Py_tp_repr: + case Py_tp_richcompare: + case Py_tp_setattr: + case Py_tp_setattro: + case Py_tp_str: + case Py_tp_traverse: + case Py_tp_getset: + case Py_tp_free: + case Py_nb_matrix_multiply: + case Py_nb_inplace_matrix_multiply: + case Py_am_await: + case Py_am_aiter: + case Py_am_anext: + case Py_tp_finalize: + case Py_am_send: + case Py_tp_vectorcall: + case Py_tp_token: + case Py_bf_getbuffer: + case Py_bf_releasebuffer: + case Py_mp_ass_subscript: + case Py_mp_length: + return _PySlot_PROBLEM_DEPRECATED; + case Py_mod_exec: + case Py_mod_abi: + return _PySlot_PROBLEM_ALLOW; + default: + return _PySlot_PROBLEM_REJECT; + } +} + +static inline _PySlot_PROBLEM_HANDLING +_PySlot_get_null_handling(uint16_t slot_id) +{ + switch (slot_id) { + case Py_slot_end: + case Py_tp_doc: + case Py_tp_token: + case Py_mod_multiple_interpreters: + case Py_mod_gil: + case Py_slot_subslots: + case Py_tp_slots: + case Py_mod_slots: + case Py_tp_basicsize: + case Py_tp_extra_basicsize: + case Py_tp_itemsize: + case Py_tp_flags: + case Py_mod_state_size: + return _PySlot_PROBLEM_ALLOW; + case Py_mp_subscript: + case Py_nb_absolute: + case Py_nb_add: + case Py_nb_and: + case Py_nb_bool: + case Py_nb_divmod: + case Py_nb_float: + case Py_nb_floor_divide: + case Py_nb_index: + case Py_nb_inplace_add: + case Py_nb_inplace_and: + case Py_nb_inplace_floor_divide: + case Py_nb_inplace_lshift: + case Py_nb_inplace_multiply: + case Py_nb_inplace_or: + case Py_nb_inplace_power: + case Py_nb_inplace_remainder: + case Py_nb_inplace_rshift: + case Py_nb_inplace_subtract: + case Py_nb_inplace_true_divide: + case Py_nb_inplace_xor: + case Py_nb_int: + case Py_nb_invert: + case Py_nb_lshift: + case Py_nb_multiply: + case Py_nb_negative: + case Py_nb_or: + case Py_nb_positive: + case Py_nb_power: + case Py_nb_remainder: + case Py_nb_rshift: + case Py_nb_subtract: + case Py_nb_true_divide: + case Py_nb_xor: + case Py_sq_ass_item: + case Py_sq_concat: + case Py_sq_contains: + case Py_sq_inplace_concat: + case Py_sq_inplace_repeat: + case Py_sq_item: + case Py_sq_length: + case Py_sq_repeat: + case Py_tp_alloc: + case Py_tp_base: + case Py_tp_bases: + case Py_tp_call: + case Py_tp_clear: + case Py_tp_dealloc: + case Py_tp_del: + case Py_tp_descr_get: + case Py_tp_descr_set: + case Py_tp_getattr: + case Py_tp_getattro: + case Py_tp_hash: + case Py_tp_init: + case Py_tp_is_gc: + case Py_tp_iter: + case Py_tp_iternext: + case Py_tp_methods: + case Py_tp_new: + case Py_tp_repr: + case Py_tp_richcompare: + case Py_tp_setattr: + case Py_tp_setattro: + case Py_tp_str: + case Py_tp_traverse: + case Py_tp_getset: + case Py_tp_free: + case Py_nb_matrix_multiply: + case Py_nb_inplace_matrix_multiply: + case Py_am_await: + case Py_am_aiter: + case Py_am_anext: + case Py_tp_finalize: + case Py_am_send: + case Py_tp_vectorcall: + case Py_mod_create: + case Py_bf_getbuffer: + case Py_bf_releasebuffer: + case Py_mp_ass_subscript: + case Py_mp_length: + return _PySlot_PROBLEM_DEPRECATED; + default: + return _PySlot_PROBLEM_REJECT; + } +} + +static inline bool +_PySlot_get_must_be_static(uint16_t slot_id) +{ + switch (slot_id) { + case Py_tp_methods: return true; + case Py_tp_members: return true; + case Py_tp_getset: return true; + case Py_mod_methods: return true; + } + return false; +} + +#endif /* _PY_HAVE_INTERNAL_SLOTS_GENERATED_H */ diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 6bf82d8322f5080..9495ccc8ac38896 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -50,27 +50,64 @@ extern "C" { CPython refcounting operations on it! */ +#define Py_INT_TAG 3 +#define Py_TAG_INVALID 2 +#define Py_TAG_REFCNT 1 +#define Py_TAG_BITS 3 -#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) +#define Py_TAGGED_SHIFT 2 -#define Py_TAG_BITS 0 +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) PyAPI_FUNC(PyObject *) _Py_stackref_get_object(_PyStackRef ref); PyAPI_FUNC(PyObject *) _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber); -PyAPI_FUNC(_PyStackRef) _Py_stackref_create(PyObject *obj, const char *filename, int linenumber); +PyAPI_FUNC(_PyStackRef) _Py_stackref_create(PyObject *obj, uint16_t flags, const char *filename, int linenumber); PyAPI_FUNC(void) _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber); +PyAPI_FUNC(_PyStackRef) _Py_stackref_get_borrowed_from(_PyStackRef ref, const char *filename, int linenumber); +PyAPI_FUNC(void) _Py_stackref_set_borrowed_from(_PyStackRef ref, _PyStackRef borrowed_from, const char *filename, int linenumber); extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref); static const _PyStackRef PyStackRef_NULL = { .index = 0 }; -static const _PyStackRef PyStackRef_ERROR = { .index = 2 }; +static const _PyStackRef PyStackRef_ERROR = { .index = (1 << Py_TAGGED_SHIFT) }; + +#define PyStackRef_None ((_PyStackRef){ .index = (2 << Py_TAGGED_SHIFT) } ) +#define _Py_STACKREF_FALSE_INDEX (3 << Py_TAGGED_SHIFT) +#define _Py_STACKREF_TRUE_INDEX (4 << Py_TAGGED_SHIFT) +#define PyStackRef_False ((_PyStackRef){ .index = _Py_STACKREF_FALSE_INDEX }) +#define PyStackRef_True ((_PyStackRef){ .index = _Py_STACKREF_TRUE_INDEX }) + +#define INITIAL_STACKREF_INDEX (5 << Py_TAGGED_SHIFT) + +#define PyStackRef_ZERO_BITS PyStackRef_NULL + +static inline _PyStackRef +PyStackRef_Wrap(void *ptr) +{ + assert(ptr != NULL); +#ifdef Py_DEBUG + assert(((uint64_t)ptr & Py_TAG_BITS) == 0); + return (_PyStackRef){ .index = ((uint64_t)ptr) | Py_TAG_INVALID }; +#else + return (_PyStackRef){ .index = (uint64_t)ptr }; +#endif +} -// Use the first 3 even numbers for None, True and False. -// Odd numbers are reserved for (tagged) integers -#define PyStackRef_None ((_PyStackRef){ .index = 4 } ) -#define PyStackRef_False ((_PyStackRef){ .index = 6 }) -#define PyStackRef_True ((_PyStackRef){ .index = 8 }) +static inline void * +PyStackRef_Unwrap(_PyStackRef ref) +{ +#ifdef Py_DEBUG + assert ((ref.index & Py_TAG_BITS) == Py_TAG_INVALID); + return (void *)(ref.index & ~Py_TAG_BITS); +#else + return (void *)(ref.index); +#endif +} -#define INITIAL_STACKREF_INDEX 10 +static inline int +PyStackRef_RefcountOnObject(_PyStackRef ref) +{ + return (ref.index & Py_TAG_REFCNT) == 0; +} static inline int PyStackRef_IsNull(_PyStackRef ref) @@ -81,7 +118,13 @@ PyStackRef_IsNull(_PyStackRef ref) static inline bool PyStackRef_IsError(_PyStackRef ref) { - return ref.index == 2; + return ref.index == (1 << Py_TAGGED_SHIFT); +} + +static inline bool +PyStackRef_IsMalformed(_PyStackRef ref) +{ + return (ref.index & Py_TAG_BITS) == Py_TAG_INVALID; } static inline bool @@ -112,7 +155,7 @@ PyStackRef_IsNone(_PyStackRef ref) static inline bool PyStackRef_IsTaggedInt(_PyStackRef ref) { - return (ref.index & 1) == 1; + return (ref.index & Py_TAG_BITS) == Py_INT_TAG; } static inline PyObject * @@ -123,50 +166,68 @@ _PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumb _Py_stackref_record_borrow(ref, filename, linenumber); return _Py_stackref_get_object(ref); } - #define PyStackRef_AsPyObjectBorrow(REF) _PyStackRef_AsPyObjectBorrow((REF), __FILE__, __LINE__) static inline PyObject * _PyStackRef_AsPyObjectSteal(_PyStackRef ref, const char *filename, int linenumber) { - return _Py_stackref_close(ref, filename, linenumber); + PyObject *obj = _Py_stackref_close(ref, filename, linenumber); + if (PyStackRef_RefcountOnObject(ref)) { + return obj; + } + return Py_NewRef(obj); } #define PyStackRef_AsPyObjectSteal(REF) _PyStackRef_AsPyObjectSteal((REF), __FILE__, __LINE__) static inline _PyStackRef _PyStackRef_FromPyObjectNew(PyObject *obj, const char *filename, int linenumber) { - Py_INCREF(obj); - return _Py_stackref_create(obj, filename, linenumber); + assert(obj != NULL); + uint16_t flags = 0; + if (!_Py_IsImmortal(obj)) { + _Py_INCREF_MORTAL(obj); + } else { + flags = Py_TAG_REFCNT; + } + return _Py_stackref_create(obj, flags, filename, linenumber); } #define PyStackRef_FromPyObjectNew(obj) _PyStackRef_FromPyObjectNew(_PyObject_CAST(obj), __FILE__, __LINE__) static inline _PyStackRef _PyStackRef_FromPyObjectSteal(PyObject *obj, const char *filename, int linenumber) { - return _Py_stackref_create(obj, filename, linenumber); + assert(obj != NULL); + uint16_t flags = 0; + if (_Py_IsImmortal(obj)) { + flags = Py_TAG_REFCNT; + } + return _Py_stackref_create(obj, flags, filename, linenumber); } #define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj), __FILE__, __LINE__) static inline _PyStackRef _PyStackRef_FromPyObjectBorrow(PyObject *obj, const char *filename, int linenumber) { - return _Py_stackref_create(obj, filename, linenumber); + return _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber); } #define PyStackRef_FromPyObjectBorrow(obj) _PyStackRef_FromPyObjectBorrow(_PyObject_CAST(obj), __FILE__, __LINE__) static inline void _PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber) { + assert(!PyStackRef_IsError(ref)); + assert(!PyStackRef_IsNull(ref)); if (PyStackRef_IsTaggedInt(ref)) { return; } PyObject *obj = _Py_stackref_close(ref, filename, linenumber); - Py_DECREF(obj); + assert(Py_REFCNT(obj) > 0); + if (PyStackRef_RefcountOnObject(ref)) { + Py_DECREF(obj); + } } #define PyStackRef_CLOSE(REF) _PyStackRef_CLOSE((REF), __FILE__, __LINE__) - static inline void _PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber) { @@ -182,31 +243,68 @@ static inline _PyStackRef _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber) { assert(!PyStackRef_IsError(ref)); + assert(!PyStackRef_IsNull(ref)); if (PyStackRef_IsTaggedInt(ref)) { return ref; } - else { - PyObject *obj = _Py_stackref_get_object(ref); + PyObject *obj = _Py_stackref_get_object(ref); + uint16_t flags = 0; + if (PyStackRef_RefcountOnObject(ref)) { Py_INCREF(obj); - return _Py_stackref_create(obj, filename, linenumber); + } else { + flags = Py_TAG_REFCNT; } + _PyStackRef new_ref = _Py_stackref_create(obj, flags, filename, linenumber); + if (flags == Py_TAG_REFCNT && !_Py_IsImmortal(obj)) { + _PyStackRef borrowed_from = _Py_stackref_get_borrowed_from(ref, filename, linenumber); + _Py_stackref_set_borrowed_from(new_ref, borrowed_from, filename, linenumber); + } + return new_ref; } #define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__) -extern void _PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *filename, int linenumber); -#define PyStackRef_CLOSE_SPECIALIZED(REF, DESTRUCT) _PyStackRef_CLOSE_SPECIALIZED(REF, DESTRUCT, __FILE__, __LINE__) - static inline _PyStackRef -PyStackRef_MakeHeapSafe(_PyStackRef ref) +_PyStackRef_DupImmortal(_PyStackRef ref, const char *filename, int linenumber) { - return ref; + assert(!PyStackRef_IsError(ref)); + assert(!PyStackRef_IsTaggedInt(ref)); + assert(!PyStackRef_RefcountOnObject(ref)); + PyObject *obj = _Py_stackref_get_object(ref); + assert(_Py_IsImmortal(obj)); + return _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber); } +#define PyStackRef_DupImmortal(REF) _PyStackRef_DupImmortal((REF), __FILE__, __LINE__) + +static inline void +_PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *filename, int linenumber) +{ + assert(!PyStackRef_IsError(ref)); + assert(!PyStackRef_IsNull(ref)); + assert(!PyStackRef_IsTaggedInt(ref)); + PyObject *obj = _Py_stackref_close(ref, filename, linenumber); + assert(Py_REFCNT(obj) > 0); + if (PyStackRef_RefcountOnObject(ref)) { + _Py_DECREF_SPECIALIZED(obj, destruct); + } +} +#define PyStackRef_CLOSE_SPECIALIZED(REF, DESTRUCT) _PyStackRef_CLOSE_SPECIALIZED(REF, DESTRUCT, __FILE__, __LINE__) static inline _PyStackRef -PyStackRef_Borrow(_PyStackRef ref) +_PyStackRef_Borrow(_PyStackRef ref, const char *filename, int linenumber) { - return PyStackRef_DUP(ref); + assert(!PyStackRef_IsError(ref)); + assert(!PyStackRef_IsNull(ref)); + if (PyStackRef_IsTaggedInt(ref)) { + return ref; + } + PyObject *obj = _Py_stackref_get_object(ref); + _PyStackRef new_ref = _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber); + if (!_Py_IsImmortal(obj)) { + _Py_stackref_set_borrowed_from(new_ref, ref, filename, linenumber); + } + return new_ref; } +#define PyStackRef_Borrow(REF) _PyStackRef_Borrow((REF), __FILE__, __LINE__) #define PyStackRef_CLEAR(REF) \ do { \ @@ -219,28 +317,56 @@ PyStackRef_Borrow(_PyStackRef ref) static inline _PyStackRef _PyStackRef_FromPyObjectStealMortal(PyObject *obj, const char *filename, int linenumber) { + assert(obj != NULL); assert(!_Py_IsImmortal(obj)); - return _Py_stackref_create(obj, filename, linenumber); + return _Py_stackref_create(obj, 0, filename, linenumber); } #define PyStackRef_FromPyObjectStealMortal(obj) _PyStackRef_FromPyObjectStealMortal(_PyObject_CAST(obj), __FILE__, __LINE__) static inline bool PyStackRef_IsHeapSafe(_PyStackRef ref) { - return true; + if ((ref.index & Py_TAG_BITS) != Py_TAG_REFCNT || PyStackRef_IsNull(ref)) { + // Tagged ints and ERROR are included. + return true; + } + + PyObject *obj = _Py_stackref_get_object(ref); + return _Py_IsImmortal(obj); } +static inline _PyStackRef +_PyStackRef_MakeHeapSafe(_PyStackRef ref, const char *filename, int linenumber) +{ + // Special references that can't be closed. + if (ref.index < INITIAL_STACKREF_INDEX) { + return ref; + } + + bool heap_safe = PyStackRef_IsHeapSafe(ref); + PyObject *obj = _Py_stackref_close(ref, filename, linenumber); + uint16_t flags = 0; + if (heap_safe) { + // Close old ref and create a new one with the same flags. + // This is necessary for correct borrow checking. + flags = ref.index & Py_TAG_BITS; + } else { + Py_INCREF(obj); + } + return _Py_stackref_create(obj, flags, filename, linenumber); +} +#define PyStackRef_MakeHeapSafe(REF) _PyStackRef_MakeHeapSafe(REF, __FILE__, __LINE__) + static inline _PyStackRef _PyStackRef_FromPyObjectNewMortal(PyObject *obj, const char *filename, int linenumber) { + assert(obj != NULL); assert(!_Py_IsStaticImmortal(obj)); Py_INCREF(obj); - return _Py_stackref_create(obj, filename, linenumber); + return _Py_stackref_create(obj, 0, filename, linenumber); } #define PyStackRef_FromPyObjectNewMortal(obj) _PyStackRef_FromPyObjectNewMortal(_PyObject_CAST(obj), __FILE__, __LINE__) -#define PyStackRef_RefcountOnObject(REF) 1 - extern int PyStackRef_Is(_PyStackRef a, _PyStackRef b); extern bool PyStackRef_IsTaggedInt(_PyStackRef ref); @@ -257,13 +383,12 @@ PyStackRef_IsNullOrInt(_PyStackRef ref); #else -#define Py_INT_TAG 3 -#define Py_TAG_INVALID 2 -#define Py_TAG_REFCNT 1 -#define Py_TAG_BITS 3 - static const _PyStackRef PyStackRef_ERROR = { .bits = Py_TAG_INVALID }; +/* For use in the JIT to clear an unused value. + * PyStackRef_ZERO_BITS has no meaning and should not be used other than by the JIT. */ +static const _PyStackRef PyStackRef_ZERO_BITS = { .bits = 0 }; + /* Wrap a pointer in a stack ref. * The resulting stack reference is not safe and should only be used * in the interpreter to pass values from one uop to another. @@ -273,6 +398,7 @@ PyStackRef_Wrap(void *ptr) { assert(ptr != NULL); #ifdef Py_DEBUG + assert(((uintptr_t)ptr & Py_TAG_BITS) == 0); return (_PyStackRef){ .bits = ((uintptr_t)ptr) | Py_TAG_INVALID }; #else return (_PyStackRef){ .bits = (uintptr_t)ptr }; @@ -296,6 +422,12 @@ PyStackRef_IsError(_PyStackRef ref) return ref.bits == Py_TAG_INVALID; } +static inline bool +PyStackRef_IsMalformed(_PyStackRef ref) +{ + return (ref.bits & Py_TAG_BITS) == Py_TAG_INVALID; +} + static inline bool PyStackRef_IsValid(_PyStackRef ref) { @@ -312,8 +444,9 @@ PyStackRef_IsTaggedInt(_PyStackRef i) static inline _PyStackRef PyStackRef_TagInt(intptr_t i) { - assert(Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (i << 2), 2) == i); - return (_PyStackRef){ .bits = ((((uintptr_t)i) << 2) | Py_INT_TAG) }; + assert(Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (intptr_t)(((uintptr_t)i) << Py_TAGGED_SHIFT), + Py_TAGGED_SHIFT) == i); + return (_PyStackRef){ .bits = ((((uintptr_t)i) << Py_TAGGED_SHIFT) | Py_INT_TAG) }; } static inline intptr_t @@ -321,7 +454,7 @@ PyStackRef_UntagInt(_PyStackRef i) { assert(PyStackRef_IsTaggedInt(i)); intptr_t val = (intptr_t)i.bits; - return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, val, 2); + return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, val, Py_TAGGED_SHIFT); } @@ -329,188 +462,9 @@ static inline _PyStackRef PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref) { assert((ref.bits & Py_TAG_BITS) == Py_INT_TAG); // Is tagged int - assert((ref.bits & (~Py_TAG_BITS)) != (INT_MAX & (~Py_TAG_BITS))); // Isn't about to overflow - return (_PyStackRef){ .bits = ref.bits + 4 }; -} - -#define PyStackRef_IsDeferredOrTaggedInt(ref) (((ref).bits & Py_TAG_REFCNT) != 0) - -#ifdef Py_GIL_DISABLED - -#define Py_TAG_DEFERRED Py_TAG_REFCNT - -#define Py_TAG_PTR ((uintptr_t)0) - - -static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED}; - -#define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits) -#define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED }) -#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED }) -#define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_DEFERRED }) - -// Checks that mask out the deferred bit in the free threading build. -#define PyStackRef_IsNone(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_None) -#define PyStackRef_IsTrue(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_True) -#define PyStackRef_IsFalse(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_False) - -#define PyStackRef_IsNullOrInt(stackref) (PyStackRef_IsNull(stackref) || PyStackRef_IsTaggedInt(stackref)) - -static inline PyObject * -PyStackRef_AsPyObjectBorrow(_PyStackRef stackref) -{ - assert(!PyStackRef_IsTaggedInt(stackref)); - PyObject *cleared = ((PyObject *)((stackref).bits & (~Py_TAG_BITS))); - return cleared; -} - -#define PyStackRef_IsDeferred(ref) (((ref).bits & Py_TAG_BITS) == Py_TAG_DEFERRED) - -static inline PyObject * -PyStackRef_NotDeferred_AsPyObject(_PyStackRef stackref) -{ - assert(!PyStackRef_IsDeferred(stackref)); - return (PyObject *)stackref.bits; -} - -static inline PyObject * -PyStackRef_AsPyObjectSteal(_PyStackRef stackref) -{ - assert(!PyStackRef_IsNull(stackref)); - if (PyStackRef_IsDeferred(stackref)) { - return Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)); - } - return PyStackRef_AsPyObjectBorrow(stackref); -} - -static inline _PyStackRef -_PyStackRef_FromPyObjectSteal(PyObject *obj) -{ - assert(obj != NULL); - // Make sure we don't take an already tagged value. - assert(((uintptr_t)obj & Py_TAG_BITS) == 0); - return (_PyStackRef){ .bits = (uintptr_t)obj }; -} -# define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj)) - -static inline bool -PyStackRef_IsHeapSafe(_PyStackRef stackref) -{ - if (PyStackRef_IsDeferred(stackref)) { - PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref); - return obj == NULL || _Py_IsImmortal(obj) || _PyObject_HasDeferredRefcount(obj); - } - return true; -} - -static inline _PyStackRef -PyStackRef_MakeHeapSafe(_PyStackRef stackref) -{ - if (PyStackRef_IsHeapSafe(stackref)) { - return stackref; - } - PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref); - return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR }; -} - -static inline _PyStackRef -PyStackRef_FromPyObjectStealMortal(PyObject *obj) -{ - assert(obj != NULL); - assert(!_Py_IsImmortal(obj)); - // Make sure we don't take an already tagged value. - assert(((uintptr_t)obj & Py_TAG_BITS) == 0); - return (_PyStackRef){ .bits = (uintptr_t)obj }; -} - -static inline _PyStackRef -PyStackRef_FromPyObjectNew(PyObject *obj) -{ - // Make sure we don't take an already tagged value. - assert(((uintptr_t)obj & Py_TAG_BITS) == 0); - assert(obj != NULL); - if (_PyObject_HasDeferredRefcount(obj)) { - return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; - } - else { - return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR }; - } -} -#define PyStackRef_FromPyObjectNew(obj) PyStackRef_FromPyObjectNew(_PyObject_CAST(obj)) - -static inline _PyStackRef -PyStackRef_FromPyObjectBorrow(PyObject *obj) -{ - // Make sure we don't take an already tagged value. - assert(((uintptr_t)obj & Py_TAG_BITS) == 0); - assert(obj != NULL); - return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; + assert((ref.bits & (~Py_TAG_BITS)) != (INTPTR_MAX & (~Py_TAG_BITS))); // Isn't about to overflow + return (_PyStackRef){ .bits = ref.bits + (1 << Py_TAGGED_SHIFT) }; } -#define PyStackRef_FromPyObjectBorrow(obj) PyStackRef_FromPyObjectBorrow(_PyObject_CAST(obj)) - -#define PyStackRef_CLOSE(REF) \ - do { \ - _PyStackRef _close_tmp = (REF); \ - assert(!PyStackRef_IsNull(_close_tmp)); \ - if (!PyStackRef_IsDeferredOrTaggedInt(_close_tmp)) { \ - Py_DECREF(PyStackRef_AsPyObjectBorrow(_close_tmp)); \ - } \ - } while (0) - -static inline void -PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct) -{ - (void)destruct; - PyStackRef_CLOSE(ref); -} - -static inline _PyStackRef -PyStackRef_DUP(_PyStackRef stackref) -{ - assert(!PyStackRef_IsNull(stackref)); - if (PyStackRef_IsDeferredOrTaggedInt(stackref)) { - return stackref; - } - Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref)); - return stackref; -} - -static inline _PyStackRef -PyStackRef_Borrow(_PyStackRef stackref) -{ - return (_PyStackRef){ .bits = stackref.bits | Py_TAG_DEFERRED }; -} - -// Convert a possibly deferred reference to a strong reference. -static inline _PyStackRef -PyStackRef_AsStrongReference(_PyStackRef stackref) -{ - return PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(stackref)); -} - -#define PyStackRef_XCLOSE(stackref) \ - do { \ - _PyStackRef _tmp = (stackref); \ - if (!PyStackRef_IsNull(_tmp)) { \ - PyStackRef_CLOSE(_tmp); \ - } \ - } while (0); - -#define PyStackRef_CLEAR(op) \ - do { \ - _PyStackRef *_tmp_op_ptr = &(op); \ - _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \ - if (!PyStackRef_IsNull(_tmp_old_op)) { \ - *_tmp_op_ptr = PyStackRef_NULL; \ - PyStackRef_CLOSE(_tmp_old_op); \ - } \ - } while (0) - -#define PyStackRef_FromPyObjectNewMortal PyStackRef_FromPyObjectNew - -#else // Py_GIL_DISABLED - -// With GIL /* References to immortal objects always have their tag bit set to Py_TAG_REFCNT * as they can (must) have their reclamation deferred */ @@ -530,13 +484,24 @@ static const _PyStackRef PyStackRef_NULL = { .bits = PyStackRef_NULL_BITS }; #define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_REFCNT }) #define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_REFCNT }) +#ifdef Py_GIL_DISABLED +// Checks that mask out the deferred bit in the free threading build. +#define PyStackRef_IsNone(REF) (((REF).bits & ~Py_TAG_REFCNT) == (uintptr_t)&_Py_NoneStruct) +#define PyStackRef_IsTrue(REF) (((REF).bits & ~Py_TAG_REFCNT) == (uintptr_t)&_Py_TrueStruct) +#define PyStackRef_IsFalse(REF) (((REF).bits & ~Py_TAG_REFCNT) == (uintptr_t)&_Py_FalseStruct) +#else #define PyStackRef_IsTrue(REF) ((REF).bits == (((uintptr_t)&_Py_TrueStruct) | Py_TAG_REFCNT)) #define PyStackRef_IsFalse(REF) ((REF).bits == (((uintptr_t)&_Py_FalseStruct) | Py_TAG_REFCNT)) #define PyStackRef_IsNone(REF) ((REF).bits == (((uintptr_t)&_Py_NoneStruct) | Py_TAG_REFCNT)) +#endif -#ifdef Py_DEBUG +#define PyStackRef_IsNullOrInt(stackref) (PyStackRef_IsNull(stackref) || PyStackRef_IsTaggedInt(stackref)) -static inline void PyStackRef_CheckValid(_PyStackRef ref) { +#if defined(Py_DEBUG) && !defined(Py_GIL_DISABLED) + +static inline void +PyStackRef_CheckValid(_PyStackRef ref) +{ assert(ref.bits != 0); int tag = ref.bits & Py_TAG_BITS; PyObject *obj = BITS_TO_PTR_MASKED(ref); @@ -587,6 +552,8 @@ PyStackRef_Borrow(_PyStackRef ref) static inline PyObject * PyStackRef_AsPyObjectSteal(_PyStackRef ref) { + assert(!PyStackRef_IsNull(ref)); + assert(!PyStackRef_IsTaggedInt(ref)); if (PyStackRef_RefcountOnObject(ref)) { return BITS_TO_PTR(ref); } @@ -599,14 +566,18 @@ static inline _PyStackRef PyStackRef_FromPyObjectSteal(PyObject *obj) { assert(obj != NULL); -#if SIZEOF_VOID_P > 4 - unsigned int tag = obj->ob_flags & Py_TAG_REFCNT; +#ifdef Py_GIL_DISABLED + return (_PyStackRef){ .bits = (uintptr_t)obj }; #else +# if SIZEOF_VOID_P > 4 + unsigned int tag = obj->ob_flags & Py_TAG_REFCNT; +# else unsigned int tag = _Py_IsImmortal(obj) ? Py_TAG_REFCNT : 0; -#endif +# endif _PyStackRef ref = ((_PyStackRef){.bits = ((uintptr_t)(obj)) | tag}); PyStackRef_CheckValid(ref); return ref; +#endif } static inline _PyStackRef @@ -614,7 +585,7 @@ PyStackRef_FromPyObjectStealMortal(PyObject *obj) { assert(obj != NULL); assert(!_Py_IsImmortal(obj)); - _PyStackRef ref = ((_PyStackRef){.bits = ((uintptr_t)(obj)) }); + _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj }; PyStackRef_CheckValid(ref); return ref; } @@ -623,9 +594,15 @@ static inline _PyStackRef _PyStackRef_FromPyObjectNew(PyObject *obj) { assert(obj != NULL); +#ifdef Py_GIL_DISABLED + if (_PyObject_HasDeferredRefcount(obj)) { + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT }; + } +#else if (_Py_IsImmortal(obj)) { - return (_PyStackRef){ .bits = ((uintptr_t)obj) | Py_TAG_REFCNT}; + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT }; } +#endif _Py_INCREF_MORTAL(obj); _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj }; PyStackRef_CheckValid(ref); @@ -648,6 +625,7 @@ _PyStackRef_FromPyObjectNewMortal(PyObject *obj) static inline _PyStackRef PyStackRef_FromPyObjectBorrow(PyObject *obj) { + assert(obj != NULL); return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT}; } @@ -667,10 +645,27 @@ PyStackRef_DUP(_PyStackRef ref) } #endif +static inline _PyStackRef +PyStackRef_DupImmortal(_PyStackRef ref) +{ + assert(!PyStackRef_IsNull(ref)); + assert(!PyStackRef_RefcountOnObject(ref)); + assert(_Py_IsImmortal(BITS_TO_PTR_MASKED(ref))); + return ref; +} + static inline bool PyStackRef_IsHeapSafe(_PyStackRef ref) { - return (ref.bits & Py_TAG_BITS) != Py_TAG_REFCNT || ref.bits == PyStackRef_NULL_BITS || _Py_IsImmortal(BITS_TO_PTR_MASKED(ref)); +#ifdef Py_GIL_DISABLED + if ((ref.bits & Py_TAG_BITS) != Py_TAG_REFCNT) { + return true; + } + PyObject *obj = BITS_TO_PTR_MASKED(ref); + return obj == NULL || _PyObject_HasDeferredRefcount(obj); +#else + return (ref.bits & Py_TAG_BITS) != Py_TAG_REFCNT || ref.bits == PyStackRef_NULL_BITS || _Py_IsImmortal(BITS_TO_PTR_MASKED(ref)); +#endif } static inline _PyStackRef @@ -686,6 +681,13 @@ PyStackRef_MakeHeapSafe(_PyStackRef ref) return ref; } +// Convert a possibly deferred reference to a strong reference. +static inline _PyStackRef +PyStackRef_AsStrongReference(_PyStackRef stackref) +{ + return PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(stackref)); +} + #ifdef _WIN32 #define PyStackRef_CLOSE(REF) \ do { \ @@ -703,12 +705,6 @@ PyStackRef_CLOSE(_PyStackRef ref) } #endif -static inline bool -PyStackRef_IsNullOrInt(_PyStackRef ref) -{ - return PyStackRef_IsNull(ref) || PyStackRef_IsTaggedInt(ref); -} - static inline void PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct) { @@ -741,8 +737,6 @@ PyStackRef_XCLOSE(_PyStackRef ref) } while (0) -#endif // Py_GIL_DISABLED - // Note: this is a macro because MSVC (Windows) has trouble inlining it. #define PyStackRef_Is(a, b) (((a).bits & (~Py_TAG_REFCNT)) == ((b).bits & (~Py_TAG_REFCNT))) @@ -799,6 +793,13 @@ _PyThreadState_PushCStackRef(PyThreadState *tstate, _PyCStackRef *ref) ref->ref = PyStackRef_NULL; } +static inline void +_PyThreadState_PushCStackRefNew(PyThreadState *tstate, _PyCStackRef *ref, PyObject *obj) +{ + _PyThreadState_PushCStackRef(tstate, ref); + ref->ref = PyStackRef_FromPyObjectNew(obj); +} + static inline void _PyThreadState_PopCStackRef(PyThreadState *tstate, _PyCStackRef *ref) { @@ -810,13 +811,24 @@ _PyThreadState_PopCStackRef(PyThreadState *tstate, _PyCStackRef *ref) PyStackRef_XCLOSE(ref->ref); } +static inline _PyStackRef +_PyThreadState_PopCStackRefSteal(PyThreadState *tstate, _PyCStackRef *ref) +{ +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + assert(tstate_impl->c_stack_refs == ref); + tstate_impl->c_stack_refs = ref->next; +#endif + return ref->ref; +} + #ifdef Py_GIL_DISABLED static inline int _Py_TryIncrefCompareStackRef(PyObject **src, PyObject *op, _PyStackRef *out) { if (_PyObject_HasDeferredRefcount(op)) { - *out = (_PyStackRef){ .bits = (uintptr_t)op | Py_TAG_DEFERRED }; + *out = (_PyStackRef){ .bits = (uintptr_t)op | Py_TAG_REFCNT }; return 1; } if (_Py_TryIncrefCompare(src, op)) { @@ -839,6 +851,13 @@ _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out) #endif +#define PyStackRef_XSETREF(dst, src) \ + do { \ + _PyStackRef _tmp_dst_ref = (dst); \ + (dst) = (src); \ + PyStackRef_XCLOSE(_tmp_dst_ref); \ + } while(0) + // Like Py_VISIT but for _PyStackRef fields #define _Py_VISIT_STACKREF(ref) \ do { \ @@ -849,6 +868,18 @@ _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out) } \ } while (0) +static inline void +_PyStackRef_CloseStack(_PyStackRef *arguments, int total_args) +{ + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } +} + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_stats.h b/Include/internal/pycore_stats.h index ab649574f33dbf1..850e6ea455227c0 100644 --- a/Include/internal/pycore_stats.h +++ b/Include/internal/pycore_stats.h @@ -15,39 +15,56 @@ extern "C" { #include "pycore_bitutils.h" // _Py_bit_length -#define STAT_INC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name++; } while (0) -#define STAT_DEC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name--; } while (0) -#define OPCODE_EXE_INC(opname) do { if (_Py_stats) _Py_stats->opcode_stats[opname].execution_count++; } while (0) -#define CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.name++; } while (0) -#define OBJECT_STAT_INC(name) do { if (_Py_stats) _Py_stats->object_stats.name++; } while (0) -#define OBJECT_STAT_INC_COND(name, cond) \ - do { if (_Py_stats && cond) _Py_stats->object_stats.name++; } while (0) -#define EVAL_CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.eval_calls[name]++; } while (0) -#define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \ - do { if (_Py_stats && PyFunction_Check(callable)) _Py_stats->call_stats.eval_calls[name]++; } while (0) -#define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0) -#define OPT_STAT_INC(name) do { if (_Py_stats) _Py_stats->optimization_stats.name++; } while (0) -#define OPT_STAT_ADD(name, n) do { if (_Py_stats) _Py_stats->optimization_stats.name += (n); } while (0) -#define UOP_STAT_INC(opname, name) do { if (_Py_stats) { assert(opname < 512); _Py_stats->optimization_stats.opcode[opname].name++; } } while (0) -#define UOP_PAIR_INC(uopcode, lastuop) \ - do { \ - if (lastuop && _Py_stats) { \ - _Py_stats->optimization_stats.opcode[lastuop].pair_count[uopcode]++; \ - } \ - lastuop = uopcode; \ +#define STAT_INC(opname, name) _Py_STATS_EXPR(opcode_stats[opname].specialization.name++) +#define STAT_DEC(opname, name) _Py_STATS_EXPR(opcode_stats[opname].specialization.name--) +#define OPCODE_EXE_INC(opname) _Py_STATS_EXPR(opcode_stats[opname].execution_count++) +#define CALL_STAT_INC(name) _Py_STATS_EXPR(call_stats.name++) +#define OBJECT_STAT_INC(name) _Py_STATS_EXPR(object_stats.name++) +#define OBJECT_STAT_INC_COND(name, cond) _Py_STATS_COND_EXPR(cond, object_stats.name++) +#define EVAL_CALL_STAT_INC(name) _Py_STATS_EXPR(call_stats.eval_calls[name]++) +#define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) _Py_STATS_COND_EXPR(PyFunction_Check(callable), call_stats.eval_calls[name]++) +#define GC_STAT_ADD(gen, name, n) _Py_STATS_EXPR(gc_stats[(gen)].name += (n)) +#define OPT_STAT_INC(name) _Py_STATS_EXPR(optimization_stats.name++) +#define OPT_STAT_ADD(name, n) _Py_STATS_EXPR(optimization_stats.name += (n)) +#define UOP_STAT_INC(opname, name) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (s) { \ + assert(opname < 512); \ + s->optimization_stats.opcode[opname].name++; \ + } \ + } while (0) +#define UOP_PAIR_INC(uopcode, lastuop) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (lastuop && s) { \ + s->optimization_stats.opcode[lastuop].pair_count[uopcode]++; \ + } \ + lastuop = uopcode; \ } while (0) -#define OPT_UNSUPPORTED_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.unsupported_opcode[opname]++; } while (0) -#define OPT_ERROR_IN_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.error_in_opcode[opname]++; } while (0) +#define OPT_UNSUPPORTED_OPCODE(opname) _Py_STATS_EXPR(optimization_stats.unsupported_opcode[opname]++) +#define OPT_ERROR_IN_OPCODE(opname) _Py_STATS_EXPR(optimization_stats.error_in_opcode[opname]++) #define OPT_HIST(length, name) \ do { \ - if (_Py_stats) { \ + PyStats *s = _PyStats_GET(); \ + if (s) { \ int bucket = _Py_bit_length(length >= 1 ? length - 1 : 0); \ bucket = (bucket >= _Py_UOP_HIST_SIZE) ? _Py_UOP_HIST_SIZE - 1 : bucket; \ - _Py_stats->optimization_stats.name[bucket]++; \ + s->optimization_stats.name[bucket]++; \ } \ } while (0) -#define RARE_EVENT_STAT_INC(name) do { if (_Py_stats) _Py_stats->rare_event_stats.name++; } while (0) -#define OPCODE_DEFERRED_INC(opname) do { if (_Py_stats && opcode == opname) _Py_stats->opcode_stats[opname].specialization.deferred++; } while (0) +#define RARE_EVENT_STAT_INC(name) _Py_STATS_EXPR(rare_event_stats.name++) +#define OPCODE_DEFERRED_INC(opname) _Py_STATS_COND_EXPR(opcode==opname, opcode_stats[opname].specialization.deferred++) + +#ifdef Py_GIL_DISABLED +#define FT_STAT_MUTEX_SLEEP_INC() _Py_STATS_EXPR(ft_stats.mutex_sleeps++) +#define FT_STAT_QSBR_POLL_INC() _Py_STATS_EXPR(ft_stats.qsbr_polls++) +#define FT_STAT_WORLD_STOP_INC() _Py_STATS_EXPR(ft_stats.world_stops++) +#else +#define FT_STAT_MUTEX_SLEEP_INC() +#define FT_STAT_QSBR_POLL_INC() +#define FT_STAT_WORLD_STOP_INC() +#endif // Export for '_opcode' shared extension PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); @@ -71,13 +88,16 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define OPT_HIST(length, name) ((void)0) #define RARE_EVENT_STAT_INC(name) ((void)0) #define OPCODE_DEFERRED_INC(opname) ((void)0) +#define FT_STAT_MUTEX_SLEEP_INC() +#define FT_STAT_QSBR_POLL_INC() +#define FT_STAT_WORLD_STOP_INC() #endif // !Py_STATS #define RARE_EVENT_INTERP_INC(interp, name) \ do { \ /* saturating add */ \ - int val = FT_ATOMIC_LOAD_UINT8_RELAXED(interp->rare_events.name); \ + uint8_t val = FT_ATOMIC_LOAD_UINT8_RELAXED(interp->rare_events.name); \ if (val < UINT8_MAX) { \ FT_ATOMIC_STORE_UINT8(interp->rare_events.name, val + 1); \ } \ @@ -90,6 +110,11 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); RARE_EVENT_INTERP_INC(interp, name); \ } while (0); \ +PyStatus _PyStats_InterpInit(PyInterpreterState *); +bool _PyStats_ThreadInit(PyInterpreterState *, _PyThreadStateImpl *); +void _PyStats_ThreadFini(_PyThreadStateImpl *); +void _PyStats_Attach(_PyThreadStateImpl *); +void _PyStats_Detach(_PyThreadStateImpl *); #ifdef __cplusplus } diff --git a/Include/internal/pycore_strhex.h b/Include/internal/pycore_strhex.h index 225f423912f2c27..656acae960ac0d2 100644 --- a/Include/internal/pycore_strhex.h +++ b/Include/internal/pycore_strhex.h @@ -10,28 +10,24 @@ extern "C" { // Returns a str() containing the hex representation of argbuf. // Export for '_hashlib' shared extension. -PyAPI_FUNC(PyObject*) _Py_strhex(const - char* argbuf, - const Py_ssize_t arglen); +PyAPI_FUNC(PyObject *) _Py_strhex(const char *argbuf, Py_ssize_t arglen); // Returns a bytes() containing the ASCII hex representation of argbuf. -extern PyObject* _Py_strhex_bytes( - const char* argbuf, - const Py_ssize_t arglen); +extern PyObject *_Py_strhex_bytes(const char *argbuf, Py_ssize_t arglen); // These variants include support for a separator between every N bytes: -extern PyObject* _Py_strhex_with_sep( - const char* argbuf, - const Py_ssize_t arglen, - PyObject* sep, - const int bytes_per_group); +extern PyObject *_Py_strhex_with_sep( + const char *argbuf, + Py_ssize_t arglen, + PyObject *sep, + Py_ssize_t bytes_per_group); // Export for 'binascii' shared extension -PyAPI_FUNC(PyObject*) _Py_strhex_bytes_with_sep( - const char* argbuf, - const Py_ssize_t arglen, - PyObject* sep, - const int bytes_per_group); +PyAPI_FUNC(PyObject *) _Py_strhex_bytes_with_sep( + const char *argbuf, + Py_ssize_t arglen, + PyObject *sep, + Py_ssize_t bytes_per_group); #ifdef __cplusplus } diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 98099b4a497b016..c650a94a1eab2e1 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -90,6 +90,7 @@ typedef struct _symtable_entry { PyObject *ste_id; /* int: key in ste_table->st_blocks */ PyObject *ste_symbols; /* dict: variable names to flags */ PyObject *ste_name; /* string: name of current block */ + PyObject *ste_function_name; /* string or NULL: for annotation blocks: name of the corresponding functions */ PyObject *ste_varnames; /* list of function parameters */ PyObject *ste_children; /* list of child blocks */ PyObject *ste_directives;/* locations of global and nonlocal statements */ @@ -126,6 +127,7 @@ typedef struct _symtable_entry { unsigned ste_method : 1; /* true if block is a function block defined in class scope */ unsigned ste_has_conditional_annotations : 1; /* true if block has conditionally executed annotations */ unsigned ste_in_conditional_block : 1; /* set while we are inside a conditionally executed block */ + unsigned ste_in_try_block : 1; /* set while we are inside a try/except block */ unsigned ste_in_unevaluated_annotation : 1; /* set while we are processing an annotation that will not be evaluated */ int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */ _Py_SourceLocation ste_loc; /* source location of block */ @@ -151,7 +153,12 @@ extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject extern void _PySymtable_Free(struct symtable *); extern PyObject *_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name); -extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); + +// Export for '_pickle' shared extension +PyAPI_FUNC(PyObject *) +_Py_Mangle(PyObject *, PyObject *); +PyAPI_FUNC(int) +_Py_IsPrivateName(PyObject *); /* Flags for def-use information */ @@ -188,7 +195,8 @@ extern struct symtable* _Py_SymtableStringObjectFlags( const char *str, PyObject *filename, int start, - PyCompilerFlags *flags); + PyCompilerFlags *flags, + PyObject *module); int _PyFuture_FromAST( struct _mod * mod, diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index 23312471c6584ef..b671225ca6ea44d 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -147,11 +147,6 @@ extern int _PyTime_FromSecondsDouble( // Clamp to [PyTime_MIN; PyTime_MAX] on overflow. extern PyTime_t _PyTime_FromMicrosecondsClamp(PyTime_t us); -// Create a timestamp from a Python int object (number of nanoseconds). -// Export for '_lsprof' shared extension. -PyAPI_FUNC(int) _PyTime_FromLong(PyTime_t *t, - PyObject *obj); - // Convert a number of seconds (Python float or int) to a timestamp. // Raise an exception and return -1 on error, return 0 on success. // Export for '_socket' shared extension. @@ -182,10 +177,6 @@ extern PyTime_t _PyTime_As100Nanoseconds(PyTime_t t, _PyTime_round_t round); #endif -// Convert a timestamp (number of nanoseconds) as a Python int object. -// Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(PyObject*) _PyTime_AsLong(PyTime_t t); - #ifndef MS_WINDOWS // Create a timestamp from a timeval structure. // Raise an exception and return -1 on overflow, return 0 on success. diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index d71dd2886999a6c..e016afaa5c56879 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -14,55 +14,6 @@ PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int, int *, P // Export for 'pyexact' shared extension PyAPI_FUNC(void) _PyTraceback_Add(const char *, const char *, int); -/* Write the Python traceback into the file 'fd'. For example: - - Traceback (most recent call first): - File "xxx", line xxx in - File "xxx", line xxx in - ... - File "xxx", line xxx in - - This function is written for debug purpose only, to dump the traceback in - the worst case: after a segmentation fault, at fatal error, etc. That's why, - it is very limited. Strings are truncated to 100 characters and encoded to - ASCII with backslashreplace. It doesn't write the source code, only the - function name, filename and line number of each frame. Write only the first - 100 frames: if the traceback is truncated, write the line " ...". - - This function is signal safe. */ - -extern void _Py_DumpTraceback( - int fd, - PyThreadState *tstate); - -/* Write the traceback of all threads into the file 'fd'. current_thread can be - NULL. - - Return NULL on success, or an error message on error. - - This function is written for debug purpose only. It calls - _Py_DumpTraceback() for each thread, and so has the same limitations. It - only write the traceback of the first 100 threads: write "..." if there are - more threads. - - If current_tstate is NULL, the function tries to get the Python thread state - of the current thread. It is not an error if the function is unable to get - the current Python thread state. - - If interp is NULL, the function tries to get the interpreter state from - the current Python thread state, or from - _PyGILState_GetInterpreterStateUnsafe() in last resort. - - It is better to pass NULL to interp and current_tstate, the function tries - different options to retrieve this information. - - This function is signal safe. */ - -extern const char* _Py_DumpTracebackThreads( - int fd, - PyInterpreterState *interp, - PyThreadState *current_tstate); - /* Write a Unicode object into the file descriptor fd. Encode the string to ASCII using the backslashreplace error handler. @@ -85,7 +36,8 @@ extern void _Py_DumpHexadecimal( uintptr_t value, Py_ssize_t width); -extern PyObject* _PyTraceBack_FromFrame( +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyTraceBack_FromFrame( PyObject *tb_next, PyFrameObject *frame); @@ -100,8 +52,11 @@ extern int _Py_WriteIndentedMargin(int, const char*, PyObject *); extern int _Py_WriteIndent(int, PyObject *); // Export for the faulthandler module +PyAPI_FUNC(void) _Py_InitDumpStack(void); PyAPI_FUNC(void) _Py_DumpStack(int fd); +extern void _Py_DumpTraceback_Init(void); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_tracemalloc.h b/Include/internal/pycore_tracemalloc.h index 572e80258763197..9974ea3c4143fa6 100644 --- a/Include/internal/pycore_tracemalloc.h +++ b/Include/internal/pycore_tracemalloc.h @@ -21,7 +21,10 @@ struct _PyTraceMalloc_Config { } initialized; /* Is tracemalloc tracing memory allocations? - Variable protected by the TABLES_LOCK(). */ + Variable protected by the TABLES_LOCK() and stored atomically. + Atomic store is used so that it can read without locking for the + general case of checking if tracemalloc is enabled. + */ int tracing; /* limit of the number of frames in a traceback, 1 by default. @@ -30,8 +33,8 @@ struct _PyTraceMalloc_Config { }; -/* Pack the frame_t structure to reduce the memory footprint on 64-bit - architectures: 12 bytes instead of 16. */ +/* Pack the tracemalloc_frame and tracemalloc_traceback structures to reduce + the memory footprint on 64-bit architectures: 12 bytes instead of 16. */ #if defined(_MSC_VER) #pragma pack(push, 4) #endif @@ -46,18 +49,22 @@ tracemalloc_frame { PyObject *filename; unsigned int lineno; }; -#ifdef _MSC_VER -#pragma pack(pop) -#endif -struct tracemalloc_traceback { +struct +#ifdef __GNUC__ +__attribute__((packed)) +#endif +tracemalloc_traceback { Py_uhash_t hash; /* Number of frames stored */ uint16_t nframe; /* Total number of frames the traceback had */ uint16_t total_nframe; - struct tracemalloc_frame frames[1]; + struct tracemalloc_frame frames[]; }; +#ifdef _MSC_VER +#pragma pack(pop) +#endif struct _tracemalloc_runtime_state { @@ -95,7 +102,7 @@ struct _tracemalloc_runtime_state { Protected by TABLES_LOCK(). */ _Py_hashtable_t *domains; - struct tracemalloc_traceback empty_traceback; + struct tracemalloc_traceback *empty_traceback; Py_tss_t reentrant_key; }; diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index bad968428c73a1a..eb2b0c84acdc7c8 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -10,9 +10,11 @@ extern "C" { #include "pycore_brc.h" // struct _brc_thread_state #include "pycore_freelist_state.h" // struct _Py_freelists +#include "pycore_interpframe_structs.h" // _PyInterpreterFrame #include "pycore_mimalloc.h" // struct _mimalloc_thread_state #include "pycore_qsbr.h" // struct qsbr - +#include "pycore_uop.h" // struct _PyUOpInstruction +#include "pycore_structs.h" #ifdef Py_GIL_DISABLED struct _gc_thread_state { @@ -21,6 +23,7 @@ struct _gc_thread_state { }; #endif + // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The // PyThreadState fields are exposed as part of the C API, although most fields // are intended to be private. The _PyThreadStateImpl fields not exposed. @@ -28,6 +31,10 @@ typedef struct _PyThreadStateImpl { // semi-public fields are in PyThreadState. PyThreadState base; + // Embedded base frame - sentinel at the bottom of the frame stack. + // Used by profiling/sampling to detect incomplete stack traces. + _PyInterpreterFrame base_frame; + // The reference count field is used to synchronize deallocation of the // thread state during runtime finalization. Py_ssize_t refcount; @@ -37,9 +44,20 @@ typedef struct _PyThreadStateImpl { uintptr_t c_stack_soft_limit; uintptr_t c_stack_hard_limit; + // PyUnstable_ThreadState_ResetStackProtection() values + uintptr_t c_stack_init_base; + uintptr_t c_stack_init_top; + PyObject *asyncio_running_loop; // Strong reference PyObject *asyncio_running_task; // Strong reference + // Distinguishes between yield and return from PyEval_EvalFrame(). + // See gen_send_ex2() in Objects/genobject.c + enum { + GENERATOR_RETURN = 0, + GENERATOR_YIELD = 1, + } generator_return_kind; + /* Head of circular linked-list of all tasks which are instances of `asyncio.Task` or subclasses of it used in `asyncio.all_tasks`. */ @@ -70,12 +88,26 @@ typedef struct _PyThreadStateImpl { // When >1, code objects do not immortalize their non-string constants. int suppress_co_const_immortalization; + +#ifdef Py_STATS + // per-thread stats, will be merged into interp->pystats_struct + PyStats *pystats_struct; // allocated by _PyStats_ThreadInit() #endif +#endif // Py_GIL_DISABLED + #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) Py_ssize_t reftotal; // this thread's total refcount operations #endif +#if _Py_TIER2 + struct _PyJitTracerState *jit_tracer_state; +#endif +#ifdef Py_GIL_DISABLED + // gh-144438: Add padding to ensure that the fields above don't share a + // cache line with other allocations. + char __padding[64]; +#endif } _PyThreadStateImpl; #ifdef __cplusplus diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index acf1bec46028acd..e294b16b3df60f6 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -21,11 +21,18 @@ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *); /* other API */ +PyAPI_FUNC(void) _PyStolenTuple_Free(PyObject *self); + #define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item) -PyAPI_FUNC(PyObject *)_PyTuple_FromArray(PyObject *const *, Py_ssize_t); PyAPI_FUNC(PyObject *)_PyTuple_FromStackRefStealOnSuccess(const union _PyStackRef *, Py_ssize_t); PyAPI_FUNC(PyObject *)_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); +PyAPI_FUNC(PyObject *) _PyTuple_BinarySlice(PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyTuple_Concat(PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyTuple_Repeat(PyObject *self, Py_ssize_t n); + +PyAPI_FUNC(PyObject *) _PyTuple_FromPair(PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyTuple_FromPairSteal(PyObject *, PyObject *); typedef struct { PyObject_HEAD diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 0ee7d555c56cdd8..785b77d3e3be81e 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -10,6 +10,7 @@ extern "C" { #include "pycore_interp_structs.h" // managed_static_type_state #include "pycore_moduleobject.h" // PyModuleObject +#include "pycore_structs.h" // _PyStackRef /* state */ @@ -25,6 +26,7 @@ extern "C" { #define _Py_TYPE_VERSION_BYTEARRAY 9 #define _Py_TYPE_VERSION_BYTES 10 #define _Py_TYPE_VERSION_COMPLEX 11 +#define _Py_TYPE_VERSION_FROZENDICT 12 #define _Py_TYPE_VERSION_NEXT 16 @@ -40,6 +42,7 @@ extern void _PyTypes_FiniTypes(PyInterpreterState *); extern void _PyTypes_FiniExtTypes(PyInterpreterState *interp); extern void _PyTypes_Fini(PyInterpreterState *); extern void _PyTypes_AfterFork(void); +extern void _PyTypes_FiniCachedDescriptors(PyInterpreterState *); static inline PyObject ** _PyStaticType_GET_WEAKREFS_LISTPTR(managed_static_type_state *state) @@ -57,7 +60,8 @@ extern void _PyStaticType_FiniBuiltin( extern void _PyStaticType_ClearWeakRefs( PyInterpreterState *interp, PyTypeObject *type); -extern managed_static_type_state * _PyStaticType_GetState( +// Exported for external JIT support +PyAPI_FUNC(managed_static_type_state *) _PyStaticType_GetState( PyInterpreterState *interp, PyTypeObject *type); @@ -89,8 +93,10 @@ _PyType_GetModuleState(PyTypeObject *type) // function PyAPI_FUNC(PyObject *) _PyType_GetDict(PyTypeObject *); +PyAPI_FUNC(PyObject *) _PyType_LookupSubclasses(PyTypeObject *); +PyAPI_FUNC(PyObject *) _PyType_InitSubclasses(PyTypeObject *); + extern PyObject * _PyType_GetBases(PyTypeObject *type); -extern PyObject * _PyType_GetMRO(PyTypeObject *type); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); extern int _PyType_HasSubclasses(PyTypeObject *); @@ -108,17 +114,25 @@ _PyType_IsReady(PyTypeObject *type) extern PyObject* _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int *suppress_missing_attribute); extern PyObject* _Py_type_getattro(PyObject *type, PyObject *name); +extern _PyStackRef _Py_type_getattro_stackref(PyTypeObject *type, PyObject *name, + int *suppress_missing_attribute); extern PyObject* _Py_BaseObject_RichCompare(PyObject* self, PyObject* other, int op); extern PyObject* _Py_slot_tp_getattro(PyObject *self, PyObject *name); extern PyObject* _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); +extern int _PyType_HasSlotTpIternext(PyTypeObject *type); + extern PyTypeObject _PyBufferWrapper_Type; PyAPI_FUNC(PyObject*) _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *meth_found); +extern PyObject *_PySuper_LookupDescr(PyTypeObject *su_type, + PyTypeObject *su_obj_type, + PyObject *name); + extern PyObject* _PyType_GetFullyQualifiedName(PyTypeObject *type, char sep); // Perform the following operation, in a thread-safe way when required by the @@ -145,8 +159,17 @@ typedef int (*_py_validate_type)(PyTypeObject *); // It will verify the ``ty`` through user-defined validation function ``validate``, // and if the validation is passed, it will set the ``tp_version`` as valid // tp_version_tag from the ``ty``. -extern int _PyType_Validate(PyTypeObject *ty, _py_validate_type validate, unsigned int *tp_version); -extern int _PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version); +// Exported for external JIT support +int _PyType_Validate(PyTypeObject *ty, _py_validate_type validate, unsigned int *tp_version); +int _PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version); + +// Precalculates count of non-unique slots and fills wrapperbase.name_count. +extern int _PyType_InitSlotDefs(PyInterpreterState *interp); + +// Like PyType_GetBaseByToken, but does not modify refcounts. +// Cannot fail; arguments must be valid. +PyAPI_FUNC(int) +_PyType_GetBaseByToken_Borrow(PyTypeObject *type, void *token, PyTypeObject **result); #ifdef __cplusplus } diff --git a/Include/internal/pycore_unicodectype.h b/Include/internal/pycore_unicodectype.h new file mode 100644 index 000000000000000..523bdb56b09cde4 --- /dev/null +++ b/Include/internal/pycore_unicodectype.h @@ -0,0 +1,25 @@ +#ifndef Py_INTERNAL_UNICODECTYPE_H +#define Py_INTERNAL_UNICODECTYPE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +extern int _PyUnicode_ToLowerFull(Py_UCS4 ch, Py_UCS4 *res); +extern int _PyUnicode_ToTitleFull(Py_UCS4 ch, Py_UCS4 *res); +extern int _PyUnicode_ToUpperFull(Py_UCS4 ch, Py_UCS4 *res); +extern int _PyUnicode_ToFoldedFull(Py_UCS4 ch, Py_UCS4 *res); +extern int _PyUnicode_IsCaseIgnorable(Py_UCS4 ch); +extern int _PyUnicode_IsCased(Py_UCS4 ch); + +// Export for 'unicodedata' shared extension. +PyAPI_FUNC(int) _PyUnicode_IsXidStart(Py_UCS4 ch); +PyAPI_FUNC(int) _PyUnicode_IsXidContinue(Py_UCS4 ch); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_UNICODECTYPE_H */ diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 3791b913c175467..75d5068f815b918 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -11,16 +11,110 @@ extern "C" { #include "pycore_fileutils.h" // _Py_error_handler #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI -/* --- Characters Type APIs ----------------------------------------------- */ -extern int _PyUnicode_IsXidStart(Py_UCS4 ch); -extern int _PyUnicode_IsXidContinue(Py_UCS4 ch); -extern int _PyUnicode_ToLowerFull(Py_UCS4 ch, Py_UCS4 *res); -extern int _PyUnicode_ToTitleFull(Py_UCS4 ch, Py_UCS4 *res); -extern int _PyUnicode_ToUpperFull(Py_UCS4 ch, Py_UCS4 *res); -extern int _PyUnicode_ToFoldedFull(Py_UCS4 ch, Py_UCS4 *res); -extern int _PyUnicode_IsCaseIgnorable(Py_UCS4 ch); -extern int _PyUnicode_IsCased(Py_UCS4 ch); +// Maximum code point of Unicode 6.0: 0x10ffff (1,114,111). +#define _Py_MAX_UNICODE 0x10ffff + + +extern int _PyUnicode_IsModifiable(PyObject *unicode); +extern void _PyUnicodeWriter_InitWithBuffer( + _PyUnicodeWriter *writer, + PyObject *buffer); +extern PyObject* _PyUnicode_Result(PyObject *unicode); +extern int _PyUnicode_DecodeUTF8Writer( + _PyUnicodeWriter *writer, + const char *s, + Py_ssize_t size, + _Py_error_handler error_handler, + const char *errors, + Py_ssize_t *consumed); +extern PyObject* _PyUnicode_ResizeCompact( + PyObject *unicode, + Py_ssize_t length); +extern PyObject* _PyUnicode_GetEmpty(void); +PyAPI_FUNC(PyObject*) _PyUnicode_BinarySlice(PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyUnicode_Repeat(PyObject *str, Py_ssize_t len); + + +/* Generic helper macro to convert characters of different types. + from_type and to_type have to be valid type names, begin and end + are pointers to the source characters which should be of type + "from_type *". to is a pointer of type "to_type *" and points to the + buffer where the result characters are written to. */ +#define _PyUnicode_CONVERT_BYTES(from_type, to_type, begin, end, to) \ + do { \ + to_type *_to = (to_type *)(to); \ + const from_type *_iter = (const from_type *)(begin);\ + const from_type *_end = (const from_type *)(end);\ + Py_ssize_t n = (_end) - (_iter); \ + const from_type *_unrolled_end = \ + _iter + _Py_SIZE_ROUND_DOWN(n, 4); \ + while (_iter < (_unrolled_end)) { \ + _to[0] = (to_type) _iter[0]; \ + _to[1] = (to_type) _iter[1]; \ + _to[2] = (to_type) _iter[2]; \ + _to[3] = (to_type) _iter[3]; \ + _iter += 4; _to += 4; \ + } \ + while (_iter < (_end)) \ + *_to++ = (to_type) *_iter++; \ + } while (0) + + +static inline void +_PyUnicode_Fill(int kind, void *data, Py_UCS4 value, + Py_ssize_t start, Py_ssize_t length) +{ + assert(0 <= start); + switch (kind) { + case PyUnicode_1BYTE_KIND: { + assert(value <= 0xff); + Py_UCS1 ch = (unsigned char)value; + Py_UCS1 *to = (Py_UCS1 *)data + start; + memset(to, ch, length); + break; + } + case PyUnicode_2BYTE_KIND: { + assert(value <= 0xffff); + Py_UCS2 ch = (Py_UCS2)value; + Py_UCS2 *to = (Py_UCS2 *)data + start; + const Py_UCS2 *end = to + length; + for (; to < end; ++to) *to = ch; + break; + } + case PyUnicode_4BYTE_KIND: { + assert(value <= _Py_MAX_UNICODE); + Py_UCS4 ch = value; + Py_UCS4 * to = (Py_UCS4 *)data + start; + const Py_UCS4 *end = to + length; + for (; to < end; ++to) *to = ch; + break; + } + default: Py_UNREACHABLE(); + } +} + +static inline int +_PyUnicode_EnsureUnicode(PyObject *obj) +{ + if (!PyUnicode_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "must be str, not %T", obj); + return -1; + } + return 0; +} + +static inline int +_PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch) +{ + assert(ch <= _Py_MAX_UNICODE); + if (_PyUnicodeWriter_Prepare(writer, 1, ch) < 0) + return -1; + PyUnicode_WRITE(writer->kind, writer->data, writer->pos, ch); + writer->pos++; + return 0; +} /* --- Unicode API -------------------------------------------------------- */ @@ -82,12 +176,16 @@ extern int _PyUnicode_FormatAdvancedWriter( Py_ssize_t start, Py_ssize_t end); +/* PyUnicodeWriter_Format, with va_list instead of `...` */ +extern int _PyUnicodeWriter_FormatV( + PyUnicodeWriter *writer, + const char *format, + va_list vargs); + /* --- UTF-7 Codecs ------------------------------------------------------- */ extern PyObject* _PyUnicode_EncodeUTF7( PyObject *unicode, /* Unicode object */ - int base64SetO, /* Encode RFC2152 Set O characters in base64 */ - int base64WhiteSpace, /* Encode whitespace (sp, ht, nl, cr) in base64 */ const char *errors); /* error handling */ /* --- UTF-8 Codecs ------------------------------------------------------- */ @@ -211,14 +309,6 @@ PyAPI_FUNC(PyObject*) _PyUnicode_JoinArray( Py_ssize_t seqlen ); -/* Test whether a unicode is equal to ASCII identifier. Return 1 if true, - 0 otherwise. The right argument must be ASCII identifier. - Any error occurs inside will be cleared before return. */ -extern int _PyUnicode_EqualToASCIIId( - PyObject *left, /* Left string */ - _Py_Identifier *right /* Right identifier */ - ); - // Test whether a unicode is equal to ASCII string. Return 1 if true, // 0 otherwise. The right argument must be ASCII-encoded string. // Any error occurs inside will be cleared before return. @@ -236,23 +326,9 @@ extern PyObject* _PyUnicode_XStrip( ); -/* Using explicit passed-in values, insert the thousands grouping - into the string pointed to by buffer. For the argument descriptions, - see Objects/stringlib/localeutil.h */ -extern Py_ssize_t _PyUnicode_InsertThousandsGrouping( - _PyUnicodeWriter *writer, - Py_ssize_t n_buffer, - PyObject *digits, - Py_ssize_t d_pos, - Py_ssize_t n_digits, - Py_ssize_t min_width, - const char *grouping, - PyObject *thousands_sep, - Py_UCS4 *maxchar, - int forward); - /* Dedent a string. - Behaviour is expected to be an exact match of `textwrap.dedent`. + Intended to dedent Python source. Unlike `textwrap.dedent`, this + only supports spaces and tabs and doesn't normalize empty lines. Return a new reference on success, NULL with exception set on error. */ extern PyObject* _PyUnicode_Dedent(PyObject *unicode); diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 6018d98d156a657..164d9d412ef5e95 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -12,10 +12,50 @@ extern "C" { static inline void _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { PyObject *string; + string = &_Py_ID(AGEN_CLOSED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(AGEN_CREATED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(AGEN_RUNNING); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(AGEN_SUSPENDED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(CANCELLED); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(CORO_CLOSED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(CORO_CREATED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(CORO_RUNNING); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(CORO_SUSPENDED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(Emax); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(Emin); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(FINISHED); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -24,6 +64,22 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(GEN_CLOSED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(GEN_CREATED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(GEN_RUNNING); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(GEN_SUSPENDED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(JSONDecodeError); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -400,6 +456,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(__lazy_import__); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(__lazy_modules__); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__le__); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -912,6 +976,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(adobe); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(after_in_child); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -932,6 +1000,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(all_interpreters); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(all_threads); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -940,6 +1012,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(alphabet); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(any); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1024,6 +1100,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(blocking); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(bound); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1092,6 +1172,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(c_stack); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(cache_frames); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cached_datetime_module); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1120,6 +1208,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(callable); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(callback); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1128,10 +1220,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(canonical); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(capath); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(capitals); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(category); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1144,10 +1244,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(chain); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(check_same_thread); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(clamp); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(clear); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1248,6 +1356,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(collector); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(command); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1260,6 +1372,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(compression); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(config); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(consts); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1300,6 +1420,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(ctx); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cwd); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1316,10 +1440,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(date); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(day); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(days); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(debug); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1436,6 +1568,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(emptyerror); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(encode); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1484,10 +1620,22 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(exc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(exc_tb); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exc_type); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(exc_val); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exc_value); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1504,6 +1652,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(exit); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exp); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1528,6 +1680,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(fallback); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(false); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1616,6 +1772,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(foldspaces); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(follow_symlinks); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1632,6 +1792,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(free_threaded); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(from_param); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1652,6 +1816,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(fullerror); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(func); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1660,6 +1828,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(gc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(generation); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1744,6 +1916,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(hours); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(id); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1760,10 +1936,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(ignorechars); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(imag); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(implieslink); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(importlib); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1772,6 +1956,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(include_aliases); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(incoming); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1956,6 +2144,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(kwargs); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(kwdefaults); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1996,6 +2188,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(legacy); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(len); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2056,6 +2252,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(mask); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(match); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2064,6 +2264,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(max_threads); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxdigits); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2080,6 +2284,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(maxsize); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxsplit); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2116,6 +2324,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(microseconds); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(milliseconds); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2124,6 +2336,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(minutes); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mod); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2144,6 +2360,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(modulo); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(month); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2192,6 +2412,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(native); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ndigits); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2296,6 +2520,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(opcodes); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(open); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2324,6 +2552,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(other); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(out_fd); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2344,6 +2576,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(pad); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(padded); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pages); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2392,6 +2632,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(pidfd); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(pointer_bits); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(policy); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2412,6 +2660,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(prec); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(preserve_exc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(print_file_and_line); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2424,6 +2680,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(progress_callback); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(progress_routine); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2444,10 +2704,22 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(qid); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(qualname); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(query); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(queuetype); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(quotetabs); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2492,6 +2764,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(recursive); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reducer_override); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2512,6 +2788,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(repeat); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(repl); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2520,6 +2800,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(repr); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(reqrefs); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(require_ready); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reserved); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2532,6 +2824,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(restrict); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(return); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2544,10 +2840,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(rounding); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(salt); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(sample_interval_us); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sched_priority); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2564,6 +2868,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(seconds); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(security_attributes); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2632,6 +2940,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(shared); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(short); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(show_cmd); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2640,6 +2956,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(signum); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(size); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2652,6 +2972,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(skip_non_matching_threads); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sleep); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2684,6 +3008,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(stack_frames); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(stacklevel); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2692,10 +3020,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(start_time_us); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(statement); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(stats); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(status); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2764,6 +3100,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(take_bytes); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(target); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2772,6 +3112,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(targetfd); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(task); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2808,6 +3152,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(third); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(threading); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2816,6 +3164,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(time); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(timeout); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2828,6 +3180,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(timespec); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(timestamp); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(timestamp_us); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(timetuple); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2852,6 +3216,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(traps); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(true); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2892,6 +3260,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(unboundop); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(unlink); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2900,6 +3272,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(updates); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(uri); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2908,6 +3284,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(utcoffset); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(value); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2952,6 +3332,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(weeks); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(which); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2964,6 +3348,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(wrapcol); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(writable); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3024,6 +3412,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(gc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_STR(anon_null); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3048,6 +3440,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(native); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_STR(anon_setcomp); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h new file mode 100644 index 000000000000000..9ff87faec40d0b4 --- /dev/null +++ b/Include/internal/pycore_uop.h @@ -0,0 +1,68 @@ +#ifndef Py_CORE_UOP_H +#define Py_CORE_UOP_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include +/* Depending on the format, + * the 32 bits between the oparg and operand are: + * UOP_FORMAT_TARGET: + * uint32_t target; + * UOP_FORMAT_JUMP + * uint16_t jump_target; + * uint16_t error_target; + */ +typedef struct _PyUOpInstruction{ + uint16_t opcode:15; + uint16_t format:1; + uint16_t oparg; + union { + uint32_t target; + struct { + uint16_t jump_target; + uint16_t error_target; + }; + }; + uint64_t operand0; // A cache entry + uint64_t operand1; +#ifdef Py_STATS + int32_t fitness; + uint64_t execution_count; +#endif +} _PyUOpInstruction; + +// Fitness is the target length of the trace we translate initially. The uop +// buffer has a small amount of extra space for entry/loop-closing overhead. +#define FITNESS_INITIAL 2500 + +#define UOP_TRACE_BUFFER_OVERHEAD 10 +#define UOP_MAX_TRACE_LENGTH (FITNESS_INITIAL + UOP_TRACE_BUFFER_OVERHEAD) + +/* Bloom filter with m = 256 + * https://en.wikipedia.org/wiki/Bloom_filter */ +#ifdef HAVE_GCC_UINT128_T +#define _Py_BLOOM_FILTER_WORDS 2 +typedef __uint128_t _Py_bloom_filter_word_t; +#else +#define _Py_BLOOM_FILTER_WORDS 4 +typedef uint64_t _Py_bloom_filter_word_t; +#endif + +#define _Py_BLOOM_FILTER_BITS_PER_WORD \ + ((int)(sizeof(_Py_bloom_filter_word_t) * 8)) +#define _Py_BLOOM_FILTER_WORD_SHIFT \ + ((sizeof(_Py_bloom_filter_word_t) == 16) ? 7 : 6) + +typedef struct { + _Py_bloom_filter_word_t bits[_Py_BLOOM_FILTER_WORDS]; +} _PyBloomFilter; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_UOP_H */ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index a9432401525ebbd..9d28c8590afcd7b 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -11,27 +11,42 @@ extern "C" { #define _EXIT_TRACE 300 #define _SET_IP 301 -#define _BINARY_OP 302 -#define _BINARY_OP_ADD_FLOAT 303 -#define _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS 304 -#define _BINARY_OP_ADD_INT 305 -#define _BINARY_OP_ADD_UNICODE 306 -#define _BINARY_OP_EXTEND 307 -#define _BINARY_OP_INPLACE_ADD_UNICODE 308 -#define _BINARY_OP_MULTIPLY_FLOAT 309 -#define _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS 310 -#define _BINARY_OP_MULTIPLY_INT 311 -#define _BINARY_OP_SUBSCR_CHECK_FUNC 312 -#define _BINARY_OP_SUBSCR_DICT 313 -#define _BINARY_OP_SUBSCR_INIT_CALL 314 -#define _BINARY_OP_SUBSCR_LIST_INT 315 -#define _BINARY_OP_SUBSCR_LIST_SLICE 316 -#define _BINARY_OP_SUBSCR_STR_INT 317 -#define _BINARY_OP_SUBSCR_TUPLE_INT 318 -#define _BINARY_OP_SUBTRACT_FLOAT 319 -#define _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS 320 -#define _BINARY_OP_SUBTRACT_INT 321 -#define _BINARY_SLICE 322 +#define _ALLOCATE_OBJECT 302 +#define _BINARY_OP 303 +#define _BINARY_OP_ADD_FLOAT 304 +#define _BINARY_OP_ADD_FLOAT_INPLACE 305 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT 306 +#define _BINARY_OP_ADD_INT 307 +#define _BINARY_OP_ADD_INT_INPLACE 308 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT 309 +#define _BINARY_OP_ADD_UNICODE 310 +#define _BINARY_OP_EXTEND 311 +#define _BINARY_OP_INPLACE_ADD_UNICODE 312 +#define _BINARY_OP_MULTIPLY_FLOAT 313 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE 314 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT 315 +#define _BINARY_OP_MULTIPLY_INT 316 +#define _BINARY_OP_MULTIPLY_INT_INPLACE 317 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT 318 +#define _BINARY_OP_SUBSCR_CHECK_FUNC 319 +#define _BINARY_OP_SUBSCR_DICT 320 +#define _BINARY_OP_SUBSCR_DICT_KNOWN_HASH 321 +#define _BINARY_OP_SUBSCR_INIT_CALL 322 +#define _BINARY_OP_SUBSCR_LIST_INT 323 +#define _BINARY_OP_SUBSCR_LIST_SLICE 324 +#define _BINARY_OP_SUBSCR_STR_INT 325 +#define _BINARY_OP_SUBSCR_TUPLE_INT 326 +#define _BINARY_OP_SUBSCR_USTR_INT 327 +#define _BINARY_OP_SUBTRACT_FLOAT 328 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE 329 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT 330 +#define _BINARY_OP_SUBTRACT_INT 331 +#define _BINARY_OP_SUBTRACT_INT_INPLACE 332 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT 333 +#define _BINARY_OP_TRUEDIV_FLOAT 334 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE 335 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT 336 +#define _BINARY_SLICE 337 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST #define _BUILD_MAP BUILD_MAP @@ -40,140 +55,198 @@ extern "C" { #define _BUILD_STRING BUILD_STRING #define _BUILD_TEMPLATE BUILD_TEMPLATE #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_BUILTIN_CLASS 323 -#define _CALL_BUILTIN_FAST 324 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 325 -#define _CALL_BUILTIN_O 326 -#define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 -#define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 -#define _CALL_ISINSTANCE 327 -#define _CALL_KW_NON_PY 328 -#define _CALL_LEN 329 -#define _CALL_LIST_APPEND 330 -#define _CALL_METHOD_DESCRIPTOR_FAST 331 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 332 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 333 -#define _CALL_METHOD_DESCRIPTOR_O 334 -#define _CALL_NON_PY_GENERAL 335 -#define _CALL_STR_1 336 -#define _CALL_TUPLE_1 337 -#define _CALL_TYPE_1 338 -#define _CHECK_AND_ALLOCATE_OBJECT 339 -#define _CHECK_ATTR_CLASS 340 -#define _CHECK_ATTR_METHOD_LAZY_DICT 341 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 342 +#define _CALL_BUILTIN_CLASS 338 +#define _CALL_BUILTIN_FAST 339 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 340 +#define _CALL_BUILTIN_O 341 +#define _CALL_FUNCTION_EX_NON_PY_GENERAL 342 +#define _CALL_INTRINSIC_1 343 +#define _CALL_INTRINSIC_2 344 +#define _CALL_ISINSTANCE 345 +#define _CALL_KW_NON_PY 346 +#define _CALL_LEN 347 +#define _CALL_LIST_APPEND 348 +#define _CALL_METHOD_DESCRIPTOR_FAST 349 +#define _CALL_METHOD_DESCRIPTOR_FAST_INLINE 350 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 351 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE 352 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 353 +#define _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE 354 +#define _CALL_METHOD_DESCRIPTOR_O 355 +#define _CALL_METHOD_DESCRIPTOR_O_INLINE 356 +#define _CALL_NON_PY_GENERAL 357 +#define _CALL_STR_1 358 +#define _CALL_TUPLE_1 359 +#define _CALL_TYPE_1 360 +#define _CHECK_ATTR_CLASS 361 +#define _CHECK_ATTR_METHOD_LAZY_DICT 362 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 363 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 343 -#define _CHECK_FUNCTION_EXACT_ARGS 344 -#define _CHECK_FUNCTION_VERSION 345 -#define _CHECK_FUNCTION_VERSION_INLINE 346 -#define _CHECK_FUNCTION_VERSION_KW 347 -#define _CHECK_IS_NOT_PY_CALLABLE 348 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 349 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 350 -#define _CHECK_METHOD_VERSION 351 -#define _CHECK_METHOD_VERSION_KW 352 -#define _CHECK_PEP_523 353 -#define _CHECK_PERIODIC 354 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 355 -#define _CHECK_RECURSION_REMAINING 356 -#define _CHECK_STACK_SPACE 357 -#define _CHECK_STACK_SPACE_OPERAND 358 -#define _CHECK_VALIDITY 359 -#define _COMPARE_OP 360 -#define _COMPARE_OP_FLOAT 361 -#define _COMPARE_OP_INT 362 -#define _COMPARE_OP_STR 363 -#define _CONTAINS_OP 364 -#define _CONTAINS_OP_DICT 365 -#define _CONTAINS_OP_SET 366 +#define _CHECK_FUNCTION_EXACT_ARGS 364 +#define _CHECK_FUNCTION_VERSION 365 +#define _CHECK_FUNCTION_VERSION_INLINE 366 +#define _CHECK_FUNCTION_VERSION_KW 367 +#define _CHECK_IS_NOT_PY_CALLABLE 368 +#define _CHECK_IS_NOT_PY_CALLABLE_EX 369 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 370 +#define _CHECK_IS_PY_CALLABLE_EX 371 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 372 +#define _CHECK_METHOD_VERSION 373 +#define _CHECK_METHOD_VERSION_KW 374 +#define _CHECK_OBJECT 375 +#define _CHECK_PEP_523 376 +#define _CHECK_PERIODIC 377 +#define _CHECK_PERIODIC_AT_END 378 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 379 +#define _CHECK_RECURSION_LIMIT 380 +#define _CHECK_RECURSION_REMAINING 381 +#define _CHECK_STACK_SPACE 382 +#define _CHECK_STACK_SPACE_OPERAND 383 +#define _CHECK_VALIDITY 384 +#define _COLD_DYNAMIC_EXIT 385 +#define _COLD_EXIT 386 +#define _COMPARE_OP 387 +#define _COMPARE_OP_FLOAT 388 +#define _COMPARE_OP_INT 389 +#define _COMPARE_OP_STR 390 +#define _CONTAINS_OP 391 +#define _CONTAINS_OP_DICT 392 +#define _CONTAINS_OP_SET 393 #define _CONVERT_VALUE CONVERT_VALUE -#define _COPY 367 -#define _COPY_1 368 -#define _COPY_2 369 -#define _COPY_3 370 +#define _COPY 394 +#define _COPY_1 395 +#define _COPY_2 396 +#define _COPY_3 397 #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 371 +#define _CREATE_INIT_FRAME 398 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 372 -#define _DICT_MERGE DICT_MERGE -#define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 373 -#define _DO_CALL_FUNCTION_EX 374 -#define _DO_CALL_KW 375 +#define _DEOPT 399 +#define _DICT_MERGE 400 +#define _DICT_UPDATE 401 +#define _DO_CALL 402 +#define _DO_CALL_FUNCTION_EX 403 +#define _DO_CALL_KW 404 +#define _DYNAMIC_EXIT 405 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 376 +#define _ERROR_POP_N 406 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 377 -#define _EXPAND_METHOD_KW 378 -#define _FATAL_ERROR 379 +#define _EXPAND_METHOD 407 +#define _EXPAND_METHOD_KW 408 +#define _FATAL_ERROR 409 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 380 -#define _FOR_ITER_GEN_FRAME 381 -#define _FOR_ITER_TIER_TWO 382 +#define _FOR_ITER 410 +#define _FOR_ITER_GEN_FRAME 411 +#define _FOR_ITER_TIER_TWO 412 +#define _FOR_ITER_VIRTUAL 413 +#define _FOR_ITER_VIRTUAL_TIER_TWO 414 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE -#define _GET_ITER GET_ITER +#define _GET_ITER 415 +#define _GET_ITER_TRAD 416 #define _GET_LEN GET_LEN -#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 383 -#define _GUARD_CALLABLE_ISINSTANCE 384 -#define _GUARD_CALLABLE_LEN 385 -#define _GUARD_CALLABLE_LIST_APPEND 386 -#define _GUARD_CALLABLE_STR_1 387 -#define _GUARD_CALLABLE_TUPLE_1 388 -#define _GUARD_CALLABLE_TYPE_1 389 -#define _GUARD_DORV_NO_DICT 390 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 391 -#define _GUARD_GLOBALS_VERSION 392 -#define _GUARD_IS_FALSE_POP 393 -#define _GUARD_IS_NONE_POP 394 -#define _GUARD_IS_NOT_NONE_POP 395 -#define _GUARD_IS_TRUE_POP 396 -#define _GUARD_KEYS_VERSION 397 -#define _GUARD_NOS_DICT 398 -#define _GUARD_NOS_FLOAT 399 -#define _GUARD_NOS_INT 400 -#define _GUARD_NOS_LIST 401 -#define _GUARD_NOS_NOT_NULL 402 -#define _GUARD_NOS_NULL 403 -#define _GUARD_NOS_OVERFLOWED 404 -#define _GUARD_NOS_TUPLE 405 -#define _GUARD_NOS_UNICODE 406 -#define _GUARD_NOT_EXHAUSTED_LIST 407 -#define _GUARD_NOT_EXHAUSTED_RANGE 408 -#define _GUARD_NOT_EXHAUSTED_TUPLE 409 -#define _GUARD_THIRD_NULL 410 -#define _GUARD_TOS_ANY_SET 411 -#define _GUARD_TOS_DICT 412 -#define _GUARD_TOS_FLOAT 413 -#define _GUARD_TOS_INT 414 -#define _GUARD_TOS_LIST 415 -#define _GUARD_TOS_OVERFLOWED 416 -#define _GUARD_TOS_SLICE 417 -#define _GUARD_TOS_TUPLE 418 -#define _GUARD_TOS_UNICODE 419 -#define _GUARD_TYPE_VERSION 420 -#define _GUARD_TYPE_VERSION_AND_LOCK 421 +#define _GUARD_3OS_ASYNC_GEN_ASEND 417 +#define _GUARD_BINARY_OP_EXTEND 418 +#define _GUARD_BINARY_OP_EXTEND_LHS 419 +#define _GUARD_BINARY_OP_EXTEND_RHS 420 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS 421 +#define _GUARD_BIT_IS_SET_POP 422 +#define _GUARD_BIT_IS_SET_POP_4 423 +#define _GUARD_BIT_IS_SET_POP_5 424 +#define _GUARD_BIT_IS_SET_POP_6 425 +#define _GUARD_BIT_IS_SET_POP_7 426 +#define _GUARD_BIT_IS_UNSET_POP 427 +#define _GUARD_BIT_IS_UNSET_POP_4 428 +#define _GUARD_BIT_IS_UNSET_POP_5 429 +#define _GUARD_BIT_IS_UNSET_POP_6 430 +#define _GUARD_BIT_IS_UNSET_POP_7 431 +#define _GUARD_CALLABLE_BUILTIN_CLASS 432 +#define _GUARD_CALLABLE_BUILTIN_FAST 433 +#define _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS 434 +#define _GUARD_CALLABLE_BUILTIN_O 435 +#define _GUARD_CALLABLE_ISINSTANCE 436 +#define _GUARD_CALLABLE_LEN 437 +#define _GUARD_CALLABLE_LIST_APPEND 438 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST 439 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 440 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS 441 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_O 442 +#define _GUARD_CALLABLE_STR_1 443 +#define _GUARD_CALLABLE_TUPLE_1 444 +#define _GUARD_CALLABLE_TYPE_1 445 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR 446 +#define _GUARD_CODE_VERSION_RETURN_VALUE 447 +#define _GUARD_CODE_VERSION_YIELD_VALUE 448 +#define _GUARD_CODE_VERSION__PUSH_FRAME 449 +#define _GUARD_DORV_NO_DICT 450 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 451 +#define _GUARD_GLOBALS_VERSION 452 +#define _GUARD_IP_RETURN_GENERATOR 453 +#define _GUARD_IP_RETURN_VALUE 454 +#define _GUARD_IP_YIELD_VALUE 455 +#define _GUARD_IP__PUSH_FRAME 456 +#define _GUARD_IS_FALSE_POP 457 +#define _GUARD_IS_NONE_POP 458 +#define _GUARD_IS_NOT_NONE_POP 459 +#define _GUARD_IS_TRUE_POP 460 +#define _GUARD_ITERATOR 461 +#define _GUARD_ITER_VIRTUAL 462 +#define _GUARD_LOAD_SUPER_ATTR_METHOD 463 +#define _GUARD_NOS_COMPACT_ASCII 464 +#define _GUARD_NOS_DICT_STORE_SUBSCRIPT 465 +#define _GUARD_NOS_DICT_SUBSCRIPT 466 +#define _GUARD_NOS_FLOAT 467 +#define _GUARD_NOS_INT 468 +#define _GUARD_NOS_ITER_VIRTUAL 469 +#define _GUARD_NOS_LIST 470 +#define _GUARD_NOS_NOT_NULL 471 +#define _GUARD_NOS_NULL 472 +#define _GUARD_NOS_OVERFLOWED 473 +#define _GUARD_NOS_TUPLE 474 +#define _GUARD_NOS_TYPE_VERSION 475 +#define _GUARD_NOS_UNICODE 476 +#define _GUARD_NOT_EXHAUSTED_LIST 477 +#define _GUARD_NOT_EXHAUSTED_RANGE 478 +#define _GUARD_NOT_EXHAUSTED_TUPLE 479 +#define _GUARD_THIRD_NULL 480 +#define _GUARD_TOS_ANY_DICT 481 +#define _GUARD_TOS_ANY_SET 482 +#define _GUARD_TOS_DICT 483 +#define _GUARD_TOS_FLOAT 484 +#define _GUARD_TOS_FROZENDICT 485 +#define _GUARD_TOS_FROZENSET 486 +#define _GUARD_TOS_INT 487 +#define _GUARD_TOS_IS_NONE 488 +#define _GUARD_TOS_LIST 489 +#define _GUARD_TOS_NOT_NULL 490 +#define _GUARD_TOS_OVERFLOWED 491 +#define _GUARD_TOS_SET 492 +#define _GUARD_TOS_SLICE 493 +#define _GUARD_TOS_TUPLE 494 +#define _GUARD_TOS_UNICODE 495 +#define _GUARD_TYPE 496 +#define _GUARD_TYPE_ITER 497 +#define _GUARD_TYPE_VERSION 498 +#define _GUARD_TYPE_VERSION_LOCKED 499 +#define _HANDLE_PENDING_AND_DEOPT 500 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 422 -#define _INIT_CALL_PY_EXACT_ARGS 423 -#define _INIT_CALL_PY_EXACT_ARGS_0 424 -#define _INIT_CALL_PY_EXACT_ARGS_1 425 -#define _INIT_CALL_PY_EXACT_ARGS_2 426 -#define _INIT_CALL_PY_EXACT_ARGS_3 427 -#define _INIT_CALL_PY_EXACT_ARGS_4 428 -#define _INSERT_NULL 429 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 501 +#define _INIT_CALL_PY_EXACT_ARGS 502 +#define _INIT_CALL_PY_EXACT_ARGS_0 503 +#define _INIT_CALL_PY_EXACT_ARGS_1 504 +#define _INIT_CALL_PY_EXACT_ARGS_2 505 +#define _INIT_CALL_PY_EXACT_ARGS_3 506 +#define _INIT_CALL_PY_EXACT_ARGS_4 507 +#define _INSERT_NULL 508 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -183,177 +256,1190 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 430 -#define _IS_OP IS_OP -#define _ITER_CHECK_LIST 431 -#define _ITER_CHECK_RANGE 432 -#define _ITER_CHECK_TUPLE 433 -#define _ITER_JUMP_LIST 434 -#define _ITER_JUMP_RANGE 435 -#define _ITER_JUMP_TUPLE 436 -#define _ITER_NEXT_LIST 437 -#define _ITER_NEXT_LIST_TIER_TWO 438 -#define _ITER_NEXT_RANGE 439 -#define _ITER_NEXT_TUPLE 440 -#define _JUMP_TO_TOP 441 +#define _IS_NONE 509 +#define _IS_OP 510 +#define _ITER_CHECK_LIST 511 +#define _ITER_CHECK_RANGE 512 +#define _ITER_CHECK_TUPLE 513 +#define _ITER_JUMP_LIST 514 +#define _ITER_JUMP_RANGE 515 +#define _ITER_JUMP_TUPLE 516 +#define _ITER_NEXT_INLINE 517 +#define _ITER_NEXT_LIST 518 +#define _ITER_NEXT_LIST_TIER_TWO 519 +#define _ITER_NEXT_RANGE 520 +#define _ITER_NEXT_TUPLE 521 +#define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT +#define _JUMP_TO_TOP 522 #define _LIST_APPEND LIST_APPEND -#define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 442 -#define _LOAD_ATTR_CLASS 443 -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 444 -#define _LOAD_ATTR_METHOD_LAZY_DICT 445 -#define _LOAD_ATTR_METHOD_NO_DICT 446 -#define _LOAD_ATTR_METHOD_WITH_VALUES 447 -#define _LOAD_ATTR_MODULE 448 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 449 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 450 -#define _LOAD_ATTR_PROPERTY_FRAME 451 -#define _LOAD_ATTR_SLOT 452 -#define _LOAD_ATTR_WITH_HINT 453 +#define _LIST_EXTEND 523 +#define _LOAD_ATTR 524 +#define _LOAD_ATTR_CLASS 525 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME 526 +#define _LOAD_ATTR_INSTANCE_VALUE 527 +#define _LOAD_ATTR_METHOD_LAZY_DICT 528 +#define _LOAD_ATTR_METHOD_NO_DICT 529 +#define _LOAD_ATTR_METHOD_WITH_VALUES 530 +#define _LOAD_ATTR_MODULE 531 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 532 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 533 +#define _LOAD_ATTR_PROPERTY_FRAME 534 +#define _LOAD_ATTR_SLOT 535 +#define _LOAD_ATTR_WITH_HINT 536 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 454 +#define _LOAD_BYTECODE 537 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 455 -#define _LOAD_CONST_INLINE_BORROW 456 -#define _LOAD_CONST_UNDER_INLINE 457 -#define _LOAD_CONST_UNDER_INLINE_BORROW 458 +#define _LOAD_CONST_INLINE 538 +#define _LOAD_CONST_INLINE_BORROW 539 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 459 -#define _LOAD_FAST_0 460 -#define _LOAD_FAST_1 461 -#define _LOAD_FAST_2 462 -#define _LOAD_FAST_3 463 -#define _LOAD_FAST_4 464 -#define _LOAD_FAST_5 465 -#define _LOAD_FAST_6 466 -#define _LOAD_FAST_7 467 +#define _LOAD_FAST 540 +#define _LOAD_FAST_0 541 +#define _LOAD_FAST_1 542 +#define _LOAD_FAST_2 543 +#define _LOAD_FAST_3 544 +#define _LOAD_FAST_4 545 +#define _LOAD_FAST_5 546 +#define _LOAD_FAST_6 547 +#define _LOAD_FAST_7 548 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 468 -#define _LOAD_FAST_BORROW_0 469 -#define _LOAD_FAST_BORROW_1 470 -#define _LOAD_FAST_BORROW_2 471 -#define _LOAD_FAST_BORROW_3 472 -#define _LOAD_FAST_BORROW_4 473 -#define _LOAD_FAST_BORROW_5 474 -#define _LOAD_FAST_BORROW_6 475 -#define _LOAD_FAST_BORROW_7 476 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW +#define _LOAD_FAST_BORROW 549 +#define _LOAD_FAST_BORROW_0 550 +#define _LOAD_FAST_BORROW_1 551 +#define _LOAD_FAST_BORROW_2 552 +#define _LOAD_FAST_BORROW_3 553 +#define _LOAD_FAST_BORROW_4 554 +#define _LOAD_FAST_BORROW_5 555 +#define _LOAD_FAST_BORROW_6 556 +#define _LOAD_FAST_BORROW_7 557 #define _LOAD_FAST_CHECK LOAD_FAST_CHECK -#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 477 -#define _LOAD_GLOBAL_BUILTINS 478 -#define _LOAD_GLOBAL_MODULE 479 +#define _LOAD_GLOBAL 558 +#define _LOAD_GLOBAL_BUILTINS 559 +#define _LOAD_GLOBAL_MODULE 560 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 480 -#define _LOAD_SMALL_INT_0 481 -#define _LOAD_SMALL_INT_1 482 -#define _LOAD_SMALL_INT_2 483 -#define _LOAD_SMALL_INT_3 484 -#define _LOAD_SPECIAL 485 +#define _LOAD_SMALL_INT 561 +#define _LOAD_SMALL_INT_0 562 +#define _LOAD_SMALL_INT_1 563 +#define _LOAD_SMALL_INT_2 564 +#define _LOAD_SMALL_INT_3 565 +#define _LOAD_SPECIAL 566 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR -#define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 486 +#define _LOAD_SUPER_ATTR_METHOD 567 +#define _LOCK_OBJECT 568 +#define _MAKE_CALLARGS_A_TUPLE 569 #define _MAKE_CELL MAKE_CELL -#define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 487 +#define _MAKE_FUNCTION 570 +#define _MAKE_HEAP_SAFE 571 +#define _MAKE_WARM 572 #define _MAP_ADD MAP_ADD -#define _MATCH_CLASS MATCH_CLASS +#define _MATCH_CLASS 573 #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 488 -#define _MAYBE_EXPAND_METHOD_KW 489 -#define _MONITOR_CALL 490 -#define _MONITOR_CALL_KW 491 -#define _MONITOR_JUMP_BACKWARD 492 -#define _MONITOR_RESUME 493 +#define _MAYBE_EXPAND_METHOD 574 +#define _MAYBE_EXPAND_METHOD_KW 575 +#define _MONITOR_CALL 576 +#define _MONITOR_CALL_KW 577 +#define _MONITOR_JUMP_BACKWARD 578 +#define _MONITOR_RESUME 579 #define _NOP NOP -#define _POP_CALL 494 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 495 -#define _POP_CALL_ONE 496 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 497 -#define _POP_CALL_TWO 498 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 499 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 500 -#define _POP_JUMP_IF_TRUE 501 +#define _POP_JUMP_IF_FALSE 580 +#define _POP_JUMP_IF_TRUE 581 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 502 -#define _POP_TOP_INT 503 -#define _POP_TOP_LOAD_CONST_INLINE 504 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 505 -#define _POP_TOP_NOP 506 -#define _POP_TOP_UNICODE 507 -#define _POP_TWO 508 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 509 +#define _POP_TOP_FLOAT 582 +#define _POP_TOP_INT 583 +#define _POP_TOP_NOP 584 +#define _POP_TOP_OPARG 585 +#define _POP_TOP_UNICODE 586 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 510 +#define _PUSH_FRAME 587 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 511 -#define _PY_FRAME_GENERAL 512 -#define _PY_FRAME_KW 513 -#define _QUICKEN_RESUME 514 -#define _REPLACE_WITH_TRUE 515 -#define _RESUME_CHECK RESUME_CHECK +#define _PUSH_NULL_CONDITIONAL 588 +#define _PUSH_TAGGED_ZERO 589 +#define _PY_FRAME_EX 590 +#define _PY_FRAME_GENERAL 591 +#define _PY_FRAME_KW 592 +#define _RECORD_3OS_GEN_FUNC 593 +#define _RECORD_4OS 594 +#define _RECORD_BOUND_METHOD 595 +#define _RECORD_CALLABLE 596 +#define _RECORD_CALLABLE_KW 597 +#define _RECORD_CODE 598 +#define _RECORD_NOS 599 +#define _RECORD_NOS_GEN_FUNC 600 +#define _RECORD_NOS_TYPE 601 +#define _RECORD_TOS 602 +#define _RECORD_TOS_TYPE 603 +#define _REPLACE_WITH_TRUE 604 +#define _RESUME_CHECK 605 #define _RETURN_GENERATOR RETURN_GENERATOR -#define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 516 -#define _SEND 517 -#define _SEND_GEN_FRAME 518 +#define _RETURN_VALUE 606 +#define _RROT_3 607 +#define _SAVE_RETURN_OFFSET 608 +#define _SEND_ASYNC_GEN 609 +#define _SEND_ASYNC_GEN_TIER_TWO 610 +#define _SEND_GEN_FRAME 611 +#define _SEND_VIRTUAL 612 +#define _SEND_VIRTUAL_TIER_TWO 613 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE -#define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 519 -#define _STORE_ATTR 520 -#define _STORE_ATTR_INSTANCE_VALUE 521 -#define _STORE_ATTR_SLOT 522 -#define _STORE_ATTR_WITH_HINT 523 +#define _SET_UPDATE 614 +#define _SPILL_OR_RELOAD 615 +#define _START_EXECUTOR 616 +#define _STORE_ATTR 617 +#define _STORE_ATTR_INSTANCE_VALUE 618 +#define _STORE_ATTR_SLOT 619 +#define _STORE_ATTR_WITH_HINT 620 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 524 -#define _STORE_FAST_0 525 -#define _STORE_FAST_1 526 -#define _STORE_FAST_2 527 -#define _STORE_FAST_3 528 -#define _STORE_FAST_4 529 -#define _STORE_FAST_5 530 -#define _STORE_FAST_6 531 -#define _STORE_FAST_7 532 -#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST -#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 533 -#define _STORE_SUBSCR 534 -#define _STORE_SUBSCR_DICT 535 -#define _STORE_SUBSCR_LIST_INT 536 -#define _SWAP 537 -#define _SWAP_2 538 -#define _SWAP_3 539 -#define _TIER2_RESUME_CHECK 540 -#define _TO_BOOL 541 +#define _STORE_SLICE 621 +#define _STORE_SUBSCR 622 +#define _STORE_SUBSCR_DICT 623 +#define _STORE_SUBSCR_DICT_KNOWN_HASH 624 +#define _STORE_SUBSCR_LIST_INT 625 +#define _SWAP 626 +#define _SWAP_2 627 +#define _SWAP_3 628 +#define _SWAP_FAST 629 +#define _SWAP_FAST_0 630 +#define _SWAP_FAST_1 631 +#define _SWAP_FAST_2 632 +#define _SWAP_FAST_3 633 +#define _SWAP_FAST_4 634 +#define _SWAP_FAST_5 635 +#define _SWAP_FAST_6 636 +#define _SWAP_FAST_7 637 +#define _TIER2_RESUME_CHECK 638 +#define _TO_BOOL 639 #define _TO_BOOL_BOOL TO_BOOL_BOOL -#define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 542 +#define _TO_BOOL_INT 640 +#define _TO_BOOL_LIST 641 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 543 -#define _UNARY_INVERT UNARY_INVERT -#define _UNARY_NEGATIVE UNARY_NEGATIVE +#define _TO_BOOL_STR 642 +#define _TRACE_RECORD TRACE_RECORD +#define _UNARY_INVERT 643 +#define _UNARY_NEGATIVE 644 +#define _UNARY_NEGATIVE_FLOAT_INPLACE 645 #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 544 -#define _UNPACK_SEQUENCE_LIST 545 -#define _UNPACK_SEQUENCE_TUPLE 546 -#define _UNPACK_SEQUENCE_TWO_TUPLE 547 +#define _UNPACK_SEQUENCE 646 +#define _UNPACK_SEQUENCE_LIST 647 +#define _UNPACK_SEQUENCE_TUPLE 648 +#define _UNPACK_SEQUENCE_TWO_TUPLE 649 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE 650 +#define _UNPACK_SEQUENCE_UNIQUE_TUPLE 651 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE 652 #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 547 +#define _YIELD_VALUE 653 +#define MAX_UOP_ID 653 +#define _ALLOCATE_OBJECT_r00 654 +#define _BINARY_OP_r23 655 +#define _BINARY_OP_ADD_FLOAT_r03 656 +#define _BINARY_OP_ADD_FLOAT_r13 657 +#define _BINARY_OP_ADD_FLOAT_r23 658 +#define _BINARY_OP_ADD_FLOAT_INPLACE_r03 659 +#define _BINARY_OP_ADD_FLOAT_INPLACE_r13 660 +#define _BINARY_OP_ADD_FLOAT_INPLACE_r23 661 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03 662 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13 663 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23 664 +#define _BINARY_OP_ADD_INT_r03 665 +#define _BINARY_OP_ADD_INT_r13 666 +#define _BINARY_OP_ADD_INT_r23 667 +#define _BINARY_OP_ADD_INT_INPLACE_r03 668 +#define _BINARY_OP_ADD_INT_INPLACE_r13 669 +#define _BINARY_OP_ADD_INT_INPLACE_r23 670 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r03 671 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r13 672 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r23 673 +#define _BINARY_OP_ADD_UNICODE_r03 674 +#define _BINARY_OP_ADD_UNICODE_r13 675 +#define _BINARY_OP_ADD_UNICODE_r23 676 +#define _BINARY_OP_EXTEND_r23 677 +#define _BINARY_OP_INPLACE_ADD_UNICODE_r21 678 +#define _BINARY_OP_MULTIPLY_FLOAT_r03 679 +#define _BINARY_OP_MULTIPLY_FLOAT_r13 680 +#define _BINARY_OP_MULTIPLY_FLOAT_r23 681 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03 682 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13 683 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23 684 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03 685 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13 686 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23 687 +#define _BINARY_OP_MULTIPLY_INT_r03 688 +#define _BINARY_OP_MULTIPLY_INT_r13 689 +#define _BINARY_OP_MULTIPLY_INT_r23 690 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_r03 691 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_r13 692 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_r23 693 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03 694 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13 695 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23 696 +#define _BINARY_OP_SUBSCR_CHECK_FUNC_r23 697 +#define _BINARY_OP_SUBSCR_DICT_r23 698 +#define _BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23 699 +#define _BINARY_OP_SUBSCR_INIT_CALL_r01 700 +#define _BINARY_OP_SUBSCR_INIT_CALL_r11 701 +#define _BINARY_OP_SUBSCR_INIT_CALL_r21 702 +#define _BINARY_OP_SUBSCR_INIT_CALL_r31 703 +#define _BINARY_OP_SUBSCR_LIST_INT_r23 704 +#define _BINARY_OP_SUBSCR_LIST_SLICE_r23 705 +#define _BINARY_OP_SUBSCR_STR_INT_r23 706 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r03 707 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r13 708 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r23 709 +#define _BINARY_OP_SUBSCR_USTR_INT_r23 710 +#define _BINARY_OP_SUBTRACT_FLOAT_r03 711 +#define _BINARY_OP_SUBTRACT_FLOAT_r13 712 +#define _BINARY_OP_SUBTRACT_FLOAT_r23 713 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03 714 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13 715 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23 716 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03 717 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13 718 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23 719 +#define _BINARY_OP_SUBTRACT_INT_r03 720 +#define _BINARY_OP_SUBTRACT_INT_r13 721 +#define _BINARY_OP_SUBTRACT_INT_r23 722 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_r03 723 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_r13 724 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_r23 725 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03 726 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13 727 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23 728 +#define _BINARY_OP_TRUEDIV_FLOAT_r23 729 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03 730 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13 731 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23 732 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03 733 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13 734 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23 735 +#define _BINARY_SLICE_r31 736 +#define _BUILD_INTERPOLATION_r01 737 +#define _BUILD_LIST_r01 738 +#define _BUILD_MAP_r01 739 +#define _BUILD_SET_r01 740 +#define _BUILD_SLICE_r01 741 +#define _BUILD_STRING_r01 742 +#define _BUILD_TEMPLATE_r21 743 +#define _BUILD_TUPLE_r01 744 +#define _CALL_BUILTIN_CLASS_r00 745 +#define _CALL_BUILTIN_FAST_r00 746 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS_r00 747 +#define _CALL_BUILTIN_O_r03 748 +#define _CALL_FUNCTION_EX_NON_PY_GENERAL_r31 749 +#define _CALL_INTRINSIC_1_r12 750 +#define _CALL_INTRINSIC_2_r23 751 +#define _CALL_ISINSTANCE_r31 752 +#define _CALL_KW_NON_PY_r11 753 +#define _CALL_LEN_r33 754 +#define _CALL_LIST_APPEND_r03 755 +#define _CALL_LIST_APPEND_r13 756 +#define _CALL_LIST_APPEND_r23 757 +#define _CALL_LIST_APPEND_r33 758 +#define _CALL_METHOD_DESCRIPTOR_FAST_r00 759 +#define _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00 760 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 761 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00 762 +#define _CALL_METHOD_DESCRIPTOR_NOARGS_r03 763 +#define _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03 764 +#define _CALL_METHOD_DESCRIPTOR_O_r03 765 +#define _CALL_METHOD_DESCRIPTOR_O_INLINE_r03 766 +#define _CALL_NON_PY_GENERAL_r01 767 +#define _CALL_STR_1_r32 768 +#define _CALL_TUPLE_1_r32 769 +#define _CALL_TYPE_1_r02 770 +#define _CALL_TYPE_1_r12 771 +#define _CALL_TYPE_1_r22 772 +#define _CALL_TYPE_1_r32 773 +#define _CHECK_ATTR_CLASS_r01 774 +#define _CHECK_ATTR_CLASS_r11 775 +#define _CHECK_ATTR_CLASS_r22 776 +#define _CHECK_ATTR_CLASS_r33 777 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r01 778 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r11 779 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r22 780 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r33 781 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00 782 +#define _CHECK_EG_MATCH_r22 783 +#define _CHECK_EXC_MATCH_r22 784 +#define _CHECK_FUNCTION_EXACT_ARGS_r00 785 +#define _CHECK_FUNCTION_VERSION_r00 786 +#define _CHECK_FUNCTION_VERSION_INLINE_r00 787 +#define _CHECK_FUNCTION_VERSION_INLINE_r11 788 +#define _CHECK_FUNCTION_VERSION_INLINE_r22 789 +#define _CHECK_FUNCTION_VERSION_INLINE_r33 790 +#define _CHECK_FUNCTION_VERSION_KW_r11 791 +#define _CHECK_IS_NOT_PY_CALLABLE_r00 792 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r03 793 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r13 794 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r23 795 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r33 796 +#define _CHECK_IS_NOT_PY_CALLABLE_KW_r11 797 +#define _CHECK_IS_PY_CALLABLE_EX_r03 798 +#define _CHECK_IS_PY_CALLABLE_EX_r13 799 +#define _CHECK_IS_PY_CALLABLE_EX_r23 800 +#define _CHECK_IS_PY_CALLABLE_EX_r33 801 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r01 802 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r11 803 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r22 804 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 805 +#define _CHECK_METHOD_VERSION_r00 806 +#define _CHECK_METHOD_VERSION_KW_r11 807 +#define _CHECK_OBJECT_r00 808 +#define _CHECK_PEP_523_r00 809 +#define _CHECK_PEP_523_r11 810 +#define _CHECK_PEP_523_r22 811 +#define _CHECK_PEP_523_r33 812 +#define _CHECK_PERIODIC_r00 813 +#define _CHECK_PERIODIC_AT_END_r00 814 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00 815 +#define _CHECK_RECURSION_LIMIT_r00 816 +#define _CHECK_RECURSION_LIMIT_r11 817 +#define _CHECK_RECURSION_LIMIT_r22 818 +#define _CHECK_RECURSION_LIMIT_r33 819 +#define _CHECK_RECURSION_REMAINING_r00 820 +#define _CHECK_RECURSION_REMAINING_r11 821 +#define _CHECK_RECURSION_REMAINING_r22 822 +#define _CHECK_RECURSION_REMAINING_r33 823 +#define _CHECK_STACK_SPACE_r00 824 +#define _CHECK_STACK_SPACE_OPERAND_r00 825 +#define _CHECK_STACK_SPACE_OPERAND_r11 826 +#define _CHECK_STACK_SPACE_OPERAND_r22 827 +#define _CHECK_STACK_SPACE_OPERAND_r33 828 +#define _CHECK_VALIDITY_r00 829 +#define _CHECK_VALIDITY_r11 830 +#define _CHECK_VALIDITY_r22 831 +#define _CHECK_VALIDITY_r33 832 +#define _COLD_DYNAMIC_EXIT_r00 833 +#define _COLD_EXIT_r00 834 +#define _COMPARE_OP_r21 835 +#define _COMPARE_OP_FLOAT_r03 836 +#define _COMPARE_OP_FLOAT_r13 837 +#define _COMPARE_OP_FLOAT_r23 838 +#define _COMPARE_OP_INT_r23 839 +#define _COMPARE_OP_STR_r23 840 +#define _CONTAINS_OP_r23 841 +#define _CONTAINS_OP_DICT_r23 842 +#define _CONTAINS_OP_SET_r23 843 +#define _CONVERT_VALUE_r11 844 +#define _COPY_r01 845 +#define _COPY_1_r02 846 +#define _COPY_1_r12 847 +#define _COPY_1_r23 848 +#define _COPY_2_r03 849 +#define _COPY_2_r13 850 +#define _COPY_2_r23 851 +#define _COPY_3_r03 852 +#define _COPY_3_r13 853 +#define _COPY_3_r23 854 +#define _COPY_3_r33 855 +#define _COPY_FREE_VARS_r00 856 +#define _COPY_FREE_VARS_r11 857 +#define _COPY_FREE_VARS_r22 858 +#define _COPY_FREE_VARS_r33 859 +#define _CREATE_INIT_FRAME_r01 860 +#define _DELETE_ATTR_r10 861 +#define _DELETE_DEREF_r00 862 +#define _DELETE_FAST_r00 863 +#define _DELETE_GLOBAL_r00 864 +#define _DELETE_NAME_r00 865 +#define _DELETE_SUBSCR_r20 866 +#define _DEOPT_r00 867 +#define _DEOPT_r10 868 +#define _DEOPT_r20 869 +#define _DEOPT_r30 870 +#define _DICT_MERGE_r11 871 +#define _DICT_UPDATE_r11 872 +#define _DO_CALL_r01 873 +#define _DO_CALL_FUNCTION_EX_r31 874 +#define _DO_CALL_KW_r11 875 +#define _DYNAMIC_EXIT_r00 876 +#define _DYNAMIC_EXIT_r10 877 +#define _DYNAMIC_EXIT_r20 878 +#define _DYNAMIC_EXIT_r30 879 +#define _END_FOR_r10 880 +#define _END_SEND_r31 881 +#define _ERROR_POP_N_r00 882 +#define _EXIT_INIT_CHECK_r10 883 +#define _EXIT_TRACE_r00 884 +#define _EXIT_TRACE_r10 885 +#define _EXIT_TRACE_r20 886 +#define _EXIT_TRACE_r30 887 +#define _EXPAND_METHOD_r00 888 +#define _EXPAND_METHOD_KW_r11 889 +#define _FATAL_ERROR_r00 890 +#define _FATAL_ERROR_r11 891 +#define _FATAL_ERROR_r22 892 +#define _FATAL_ERROR_r33 893 +#define _FORMAT_SIMPLE_r11 894 +#define _FORMAT_WITH_SPEC_r21 895 +#define _FOR_ITER_r23 896 +#define _FOR_ITER_GEN_FRAME_r03 897 +#define _FOR_ITER_GEN_FRAME_r13 898 +#define _FOR_ITER_GEN_FRAME_r23 899 +#define _FOR_ITER_TIER_TWO_r23 900 +#define _FOR_ITER_VIRTUAL_r23 901 +#define _FOR_ITER_VIRTUAL_TIER_TWO_r23 902 +#define _GET_AITER_r11 903 +#define _GET_ANEXT_r12 904 +#define _GET_AWAITABLE_r11 905 +#define _GET_ITER_r12 906 +#define _GET_ITER_TRAD_r12 907 +#define _GET_LEN_r12 908 +#define _GUARD_3OS_ASYNC_GEN_ASEND_r03 909 +#define _GUARD_3OS_ASYNC_GEN_ASEND_r13 910 +#define _GUARD_3OS_ASYNC_GEN_ASEND_r23 911 +#define _GUARD_3OS_ASYNC_GEN_ASEND_r33 912 +#define _GUARD_BINARY_OP_EXTEND_r22 913 +#define _GUARD_BINARY_OP_EXTEND_LHS_r02 914 +#define _GUARD_BINARY_OP_EXTEND_LHS_r12 915 +#define _GUARD_BINARY_OP_EXTEND_LHS_r22 916 +#define _GUARD_BINARY_OP_EXTEND_LHS_r33 917 +#define _GUARD_BINARY_OP_EXTEND_RHS_r02 918 +#define _GUARD_BINARY_OP_EXTEND_RHS_r12 919 +#define _GUARD_BINARY_OP_EXTEND_RHS_r22 920 +#define _GUARD_BINARY_OP_EXTEND_RHS_r33 921 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02 922 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12 923 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22 924 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33 925 +#define _GUARD_BIT_IS_SET_POP_r00 926 +#define _GUARD_BIT_IS_SET_POP_r10 927 +#define _GUARD_BIT_IS_SET_POP_r21 928 +#define _GUARD_BIT_IS_SET_POP_r32 929 +#define _GUARD_BIT_IS_SET_POP_4_r00 930 +#define _GUARD_BIT_IS_SET_POP_4_r10 931 +#define _GUARD_BIT_IS_SET_POP_4_r21 932 +#define _GUARD_BIT_IS_SET_POP_4_r32 933 +#define _GUARD_BIT_IS_SET_POP_5_r00 934 +#define _GUARD_BIT_IS_SET_POP_5_r10 935 +#define _GUARD_BIT_IS_SET_POP_5_r21 936 +#define _GUARD_BIT_IS_SET_POP_5_r32 937 +#define _GUARD_BIT_IS_SET_POP_6_r00 938 +#define _GUARD_BIT_IS_SET_POP_6_r10 939 +#define _GUARD_BIT_IS_SET_POP_6_r21 940 +#define _GUARD_BIT_IS_SET_POP_6_r32 941 +#define _GUARD_BIT_IS_SET_POP_7_r00 942 +#define _GUARD_BIT_IS_SET_POP_7_r10 943 +#define _GUARD_BIT_IS_SET_POP_7_r21 944 +#define _GUARD_BIT_IS_SET_POP_7_r32 945 +#define _GUARD_BIT_IS_UNSET_POP_r00 946 +#define _GUARD_BIT_IS_UNSET_POP_r10 947 +#define _GUARD_BIT_IS_UNSET_POP_r21 948 +#define _GUARD_BIT_IS_UNSET_POP_r32 949 +#define _GUARD_BIT_IS_UNSET_POP_4_r00 950 +#define _GUARD_BIT_IS_UNSET_POP_4_r10 951 +#define _GUARD_BIT_IS_UNSET_POP_4_r21 952 +#define _GUARD_BIT_IS_UNSET_POP_4_r32 953 +#define _GUARD_BIT_IS_UNSET_POP_5_r00 954 +#define _GUARD_BIT_IS_UNSET_POP_5_r10 955 +#define _GUARD_BIT_IS_UNSET_POP_5_r21 956 +#define _GUARD_BIT_IS_UNSET_POP_5_r32 957 +#define _GUARD_BIT_IS_UNSET_POP_6_r00 958 +#define _GUARD_BIT_IS_UNSET_POP_6_r10 959 +#define _GUARD_BIT_IS_UNSET_POP_6_r21 960 +#define _GUARD_BIT_IS_UNSET_POP_6_r32 961 +#define _GUARD_BIT_IS_UNSET_POP_7_r00 962 +#define _GUARD_BIT_IS_UNSET_POP_7_r10 963 +#define _GUARD_BIT_IS_UNSET_POP_7_r21 964 +#define _GUARD_BIT_IS_UNSET_POP_7_r32 965 +#define _GUARD_CALLABLE_BUILTIN_CLASS_r00 966 +#define _GUARD_CALLABLE_BUILTIN_FAST_r00 967 +#define _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00 968 +#define _GUARD_CALLABLE_BUILTIN_O_r00 969 +#define _GUARD_CALLABLE_ISINSTANCE_r03 970 +#define _GUARD_CALLABLE_ISINSTANCE_r13 971 +#define _GUARD_CALLABLE_ISINSTANCE_r23 972 +#define _GUARD_CALLABLE_ISINSTANCE_r33 973 +#define _GUARD_CALLABLE_LEN_r03 974 +#define _GUARD_CALLABLE_LEN_r13 975 +#define _GUARD_CALLABLE_LEN_r23 976 +#define _GUARD_CALLABLE_LEN_r33 977 +#define _GUARD_CALLABLE_LIST_APPEND_r03 978 +#define _GUARD_CALLABLE_LIST_APPEND_r13 979 +#define _GUARD_CALLABLE_LIST_APPEND_r23 980 +#define _GUARD_CALLABLE_LIST_APPEND_r33 981 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00 982 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 983 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00 984 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00 985 +#define _GUARD_CALLABLE_STR_1_r03 986 +#define _GUARD_CALLABLE_STR_1_r13 987 +#define _GUARD_CALLABLE_STR_1_r23 988 +#define _GUARD_CALLABLE_STR_1_r33 989 +#define _GUARD_CALLABLE_TUPLE_1_r03 990 +#define _GUARD_CALLABLE_TUPLE_1_r13 991 +#define _GUARD_CALLABLE_TUPLE_1_r23 992 +#define _GUARD_CALLABLE_TUPLE_1_r33 993 +#define _GUARD_CALLABLE_TYPE_1_r03 994 +#define _GUARD_CALLABLE_TYPE_1_r13 995 +#define _GUARD_CALLABLE_TYPE_1_r23 996 +#define _GUARD_CALLABLE_TYPE_1_r33 997 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r00 998 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r11 999 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r22 1000 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r33 1001 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r00 1002 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r11 1003 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r22 1004 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r33 1005 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r00 1006 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r11 1007 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r22 1008 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r33 1009 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r00 1010 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r11 1011 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r22 1012 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r33 1013 +#define _GUARD_DORV_NO_DICT_r01 1014 +#define _GUARD_DORV_NO_DICT_r11 1015 +#define _GUARD_DORV_NO_DICT_r22 1016 +#define _GUARD_DORV_NO_DICT_r33 1017 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01 1018 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 1019 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 1020 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 1021 +#define _GUARD_GLOBALS_VERSION_r00 1022 +#define _GUARD_GLOBALS_VERSION_r11 1023 +#define _GUARD_GLOBALS_VERSION_r22 1024 +#define _GUARD_GLOBALS_VERSION_r33 1025 +#define _GUARD_IP_RETURN_GENERATOR_r00 1026 +#define _GUARD_IP_RETURN_GENERATOR_r11 1027 +#define _GUARD_IP_RETURN_GENERATOR_r22 1028 +#define _GUARD_IP_RETURN_GENERATOR_r33 1029 +#define _GUARD_IP_RETURN_VALUE_r00 1030 +#define _GUARD_IP_RETURN_VALUE_r11 1031 +#define _GUARD_IP_RETURN_VALUE_r22 1032 +#define _GUARD_IP_RETURN_VALUE_r33 1033 +#define _GUARD_IP_YIELD_VALUE_r00 1034 +#define _GUARD_IP_YIELD_VALUE_r11 1035 +#define _GUARD_IP_YIELD_VALUE_r22 1036 +#define _GUARD_IP_YIELD_VALUE_r33 1037 +#define _GUARD_IP__PUSH_FRAME_r00 1038 +#define _GUARD_IP__PUSH_FRAME_r11 1039 +#define _GUARD_IP__PUSH_FRAME_r22 1040 +#define _GUARD_IP__PUSH_FRAME_r33 1041 +#define _GUARD_IS_FALSE_POP_r00 1042 +#define _GUARD_IS_FALSE_POP_r10 1043 +#define _GUARD_IS_FALSE_POP_r21 1044 +#define _GUARD_IS_FALSE_POP_r32 1045 +#define _GUARD_IS_NONE_POP_r00 1046 +#define _GUARD_IS_NONE_POP_r10 1047 +#define _GUARD_IS_NONE_POP_r21 1048 +#define _GUARD_IS_NONE_POP_r32 1049 +#define _GUARD_IS_NOT_NONE_POP_r10 1050 +#define _GUARD_IS_TRUE_POP_r00 1051 +#define _GUARD_IS_TRUE_POP_r10 1052 +#define _GUARD_IS_TRUE_POP_r21 1053 +#define _GUARD_IS_TRUE_POP_r32 1054 +#define _GUARD_ITERATOR_r01 1055 +#define _GUARD_ITERATOR_r11 1056 +#define _GUARD_ITERATOR_r22 1057 +#define _GUARD_ITERATOR_r33 1058 +#define _GUARD_ITER_VIRTUAL_r01 1059 +#define _GUARD_ITER_VIRTUAL_r11 1060 +#define _GUARD_ITER_VIRTUAL_r22 1061 +#define _GUARD_ITER_VIRTUAL_r33 1062 +#define _GUARD_LOAD_SUPER_ATTR_METHOD_r03 1063 +#define _GUARD_LOAD_SUPER_ATTR_METHOD_r13 1064 +#define _GUARD_LOAD_SUPER_ATTR_METHOD_r23 1065 +#define _GUARD_LOAD_SUPER_ATTR_METHOD_r33 1066 +#define _GUARD_NOS_COMPACT_ASCII_r02 1067 +#define _GUARD_NOS_COMPACT_ASCII_r12 1068 +#define _GUARD_NOS_COMPACT_ASCII_r22 1069 +#define _GUARD_NOS_COMPACT_ASCII_r33 1070 +#define _GUARD_NOS_DICT_STORE_SUBSCRIPT_r03 1071 +#define _GUARD_NOS_DICT_STORE_SUBSCRIPT_r13 1072 +#define _GUARD_NOS_DICT_STORE_SUBSCRIPT_r23 1073 +#define _GUARD_NOS_DICT_STORE_SUBSCRIPT_r33 1074 +#define _GUARD_NOS_DICT_SUBSCRIPT_r02 1075 +#define _GUARD_NOS_DICT_SUBSCRIPT_r12 1076 +#define _GUARD_NOS_DICT_SUBSCRIPT_r22 1077 +#define _GUARD_NOS_DICT_SUBSCRIPT_r33 1078 +#define _GUARD_NOS_FLOAT_r02 1079 +#define _GUARD_NOS_FLOAT_r12 1080 +#define _GUARD_NOS_FLOAT_r22 1081 +#define _GUARD_NOS_FLOAT_r33 1082 +#define _GUARD_NOS_INT_r02 1083 +#define _GUARD_NOS_INT_r12 1084 +#define _GUARD_NOS_INT_r22 1085 +#define _GUARD_NOS_INT_r33 1086 +#define _GUARD_NOS_ITER_VIRTUAL_r02 1087 +#define _GUARD_NOS_ITER_VIRTUAL_r12 1088 +#define _GUARD_NOS_ITER_VIRTUAL_r22 1089 +#define _GUARD_NOS_ITER_VIRTUAL_r33 1090 +#define _GUARD_NOS_LIST_r02 1091 +#define _GUARD_NOS_LIST_r12 1092 +#define _GUARD_NOS_LIST_r22 1093 +#define _GUARD_NOS_LIST_r33 1094 +#define _GUARD_NOS_NOT_NULL_r02 1095 +#define _GUARD_NOS_NOT_NULL_r12 1096 +#define _GUARD_NOS_NOT_NULL_r22 1097 +#define _GUARD_NOS_NOT_NULL_r33 1098 +#define _GUARD_NOS_NULL_r02 1099 +#define _GUARD_NOS_NULL_r12 1100 +#define _GUARD_NOS_NULL_r22 1101 +#define _GUARD_NOS_NULL_r33 1102 +#define _GUARD_NOS_OVERFLOWED_r02 1103 +#define _GUARD_NOS_OVERFLOWED_r12 1104 +#define _GUARD_NOS_OVERFLOWED_r22 1105 +#define _GUARD_NOS_OVERFLOWED_r33 1106 +#define _GUARD_NOS_TUPLE_r02 1107 +#define _GUARD_NOS_TUPLE_r12 1108 +#define _GUARD_NOS_TUPLE_r22 1109 +#define _GUARD_NOS_TUPLE_r33 1110 +#define _GUARD_NOS_TYPE_VERSION_r02 1111 +#define _GUARD_NOS_TYPE_VERSION_r12 1112 +#define _GUARD_NOS_TYPE_VERSION_r22 1113 +#define _GUARD_NOS_TYPE_VERSION_r33 1114 +#define _GUARD_NOS_UNICODE_r02 1115 +#define _GUARD_NOS_UNICODE_r12 1116 +#define _GUARD_NOS_UNICODE_r22 1117 +#define _GUARD_NOS_UNICODE_r33 1118 +#define _GUARD_NOT_EXHAUSTED_LIST_r02 1119 +#define _GUARD_NOT_EXHAUSTED_LIST_r12 1120 +#define _GUARD_NOT_EXHAUSTED_LIST_r22 1121 +#define _GUARD_NOT_EXHAUSTED_LIST_r33 1122 +#define _GUARD_NOT_EXHAUSTED_RANGE_r02 1123 +#define _GUARD_NOT_EXHAUSTED_RANGE_r12 1124 +#define _GUARD_NOT_EXHAUSTED_RANGE_r22 1125 +#define _GUARD_NOT_EXHAUSTED_RANGE_r33 1126 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r02 1127 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r12 1128 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 1129 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 1130 +#define _GUARD_THIRD_NULL_r03 1131 +#define _GUARD_THIRD_NULL_r13 1132 +#define _GUARD_THIRD_NULL_r23 1133 +#define _GUARD_THIRD_NULL_r33 1134 +#define _GUARD_TOS_ANY_DICT_r01 1135 +#define _GUARD_TOS_ANY_DICT_r11 1136 +#define _GUARD_TOS_ANY_DICT_r22 1137 +#define _GUARD_TOS_ANY_DICT_r33 1138 +#define _GUARD_TOS_ANY_SET_r01 1139 +#define _GUARD_TOS_ANY_SET_r11 1140 +#define _GUARD_TOS_ANY_SET_r22 1141 +#define _GUARD_TOS_ANY_SET_r33 1142 +#define _GUARD_TOS_DICT_r01 1143 +#define _GUARD_TOS_DICT_r11 1144 +#define _GUARD_TOS_DICT_r22 1145 +#define _GUARD_TOS_DICT_r33 1146 +#define _GUARD_TOS_FLOAT_r01 1147 +#define _GUARD_TOS_FLOAT_r11 1148 +#define _GUARD_TOS_FLOAT_r22 1149 +#define _GUARD_TOS_FLOAT_r33 1150 +#define _GUARD_TOS_FROZENDICT_r01 1151 +#define _GUARD_TOS_FROZENDICT_r11 1152 +#define _GUARD_TOS_FROZENDICT_r22 1153 +#define _GUARD_TOS_FROZENDICT_r33 1154 +#define _GUARD_TOS_FROZENSET_r01 1155 +#define _GUARD_TOS_FROZENSET_r11 1156 +#define _GUARD_TOS_FROZENSET_r22 1157 +#define _GUARD_TOS_FROZENSET_r33 1158 +#define _GUARD_TOS_INT_r01 1159 +#define _GUARD_TOS_INT_r11 1160 +#define _GUARD_TOS_INT_r22 1161 +#define _GUARD_TOS_INT_r33 1162 +#define _GUARD_TOS_IS_NONE_r01 1163 +#define _GUARD_TOS_IS_NONE_r11 1164 +#define _GUARD_TOS_IS_NONE_r22 1165 +#define _GUARD_TOS_IS_NONE_r33 1166 +#define _GUARD_TOS_LIST_r01 1167 +#define _GUARD_TOS_LIST_r11 1168 +#define _GUARD_TOS_LIST_r22 1169 +#define _GUARD_TOS_LIST_r33 1170 +#define _GUARD_TOS_NOT_NULL_r01 1171 +#define _GUARD_TOS_NOT_NULL_r11 1172 +#define _GUARD_TOS_NOT_NULL_r22 1173 +#define _GUARD_TOS_NOT_NULL_r33 1174 +#define _GUARD_TOS_OVERFLOWED_r01 1175 +#define _GUARD_TOS_OVERFLOWED_r11 1176 +#define _GUARD_TOS_OVERFLOWED_r22 1177 +#define _GUARD_TOS_OVERFLOWED_r33 1178 +#define _GUARD_TOS_SET_r01 1179 +#define _GUARD_TOS_SET_r11 1180 +#define _GUARD_TOS_SET_r22 1181 +#define _GUARD_TOS_SET_r33 1182 +#define _GUARD_TOS_SLICE_r01 1183 +#define _GUARD_TOS_SLICE_r11 1184 +#define _GUARD_TOS_SLICE_r22 1185 +#define _GUARD_TOS_SLICE_r33 1186 +#define _GUARD_TOS_TUPLE_r01 1187 +#define _GUARD_TOS_TUPLE_r11 1188 +#define _GUARD_TOS_TUPLE_r22 1189 +#define _GUARD_TOS_TUPLE_r33 1190 +#define _GUARD_TOS_UNICODE_r01 1191 +#define _GUARD_TOS_UNICODE_r11 1192 +#define _GUARD_TOS_UNICODE_r22 1193 +#define _GUARD_TOS_UNICODE_r33 1194 +#define _GUARD_TYPE_r01 1195 +#define _GUARD_TYPE_r11 1196 +#define _GUARD_TYPE_r22 1197 +#define _GUARD_TYPE_r33 1198 +#define _GUARD_TYPE_ITER_r02 1199 +#define _GUARD_TYPE_ITER_r12 1200 +#define _GUARD_TYPE_ITER_r22 1201 +#define _GUARD_TYPE_ITER_r33 1202 +#define _GUARD_TYPE_VERSION_r01 1203 +#define _GUARD_TYPE_VERSION_r11 1204 +#define _GUARD_TYPE_VERSION_r22 1205 +#define _GUARD_TYPE_VERSION_r33 1206 +#define _GUARD_TYPE_VERSION_LOCKED_r01 1207 +#define _GUARD_TYPE_VERSION_LOCKED_r11 1208 +#define _GUARD_TYPE_VERSION_LOCKED_r22 1209 +#define _GUARD_TYPE_VERSION_LOCKED_r33 1210 +#define _HANDLE_PENDING_AND_DEOPT_r00 1211 +#define _HANDLE_PENDING_AND_DEOPT_r10 1212 +#define _HANDLE_PENDING_AND_DEOPT_r20 1213 +#define _HANDLE_PENDING_AND_DEOPT_r30 1214 +#define _IMPORT_FROM_r12 1215 +#define _IMPORT_NAME_r21 1216 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 1217 +#define _INIT_CALL_PY_EXACT_ARGS_r01 1218 +#define _INIT_CALL_PY_EXACT_ARGS_0_r01 1219 +#define _INIT_CALL_PY_EXACT_ARGS_1_r01 1220 +#define _INIT_CALL_PY_EXACT_ARGS_2_r01 1221 +#define _INIT_CALL_PY_EXACT_ARGS_3_r01 1222 +#define _INIT_CALL_PY_EXACT_ARGS_4_r01 1223 +#define _INSERT_NULL_r10 1224 +#define _INSTRUMENTED_FOR_ITER_r23 1225 +#define _INSTRUMENTED_INSTRUCTION_r00 1226 +#define _INSTRUMENTED_JUMP_FORWARD_r00 1227 +#define _INSTRUMENTED_JUMP_FORWARD_r11 1228 +#define _INSTRUMENTED_JUMP_FORWARD_r22 1229 +#define _INSTRUMENTED_JUMP_FORWARD_r33 1230 +#define _INSTRUMENTED_LINE_r00 1231 +#define _INSTRUMENTED_NOT_TAKEN_r00 1232 +#define _INSTRUMENTED_NOT_TAKEN_r11 1233 +#define _INSTRUMENTED_NOT_TAKEN_r22 1234 +#define _INSTRUMENTED_NOT_TAKEN_r33 1235 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 1236 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 1237 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 1238 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 1239 +#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 1240 +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 1241 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 1242 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 1243 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 1244 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 1245 +#define _IS_NONE_r11 1246 +#define _IS_OP_r03 1247 +#define _IS_OP_r13 1248 +#define _IS_OP_r23 1249 +#define _ITER_CHECK_LIST_r02 1250 +#define _ITER_CHECK_LIST_r12 1251 +#define _ITER_CHECK_LIST_r22 1252 +#define _ITER_CHECK_LIST_r33 1253 +#define _ITER_CHECK_RANGE_r02 1254 +#define _ITER_CHECK_RANGE_r12 1255 +#define _ITER_CHECK_RANGE_r22 1256 +#define _ITER_CHECK_RANGE_r33 1257 +#define _ITER_CHECK_TUPLE_r02 1258 +#define _ITER_CHECK_TUPLE_r12 1259 +#define _ITER_CHECK_TUPLE_r22 1260 +#define _ITER_CHECK_TUPLE_r33 1261 +#define _ITER_JUMP_LIST_r02 1262 +#define _ITER_JUMP_LIST_r12 1263 +#define _ITER_JUMP_LIST_r22 1264 +#define _ITER_JUMP_LIST_r33 1265 +#define _ITER_JUMP_RANGE_r02 1266 +#define _ITER_JUMP_RANGE_r12 1267 +#define _ITER_JUMP_RANGE_r22 1268 +#define _ITER_JUMP_RANGE_r33 1269 +#define _ITER_JUMP_TUPLE_r02 1270 +#define _ITER_JUMP_TUPLE_r12 1271 +#define _ITER_JUMP_TUPLE_r22 1272 +#define _ITER_JUMP_TUPLE_r33 1273 +#define _ITER_NEXT_INLINE_r23 1274 +#define _ITER_NEXT_LIST_r23 1275 +#define _ITER_NEXT_LIST_TIER_TWO_r23 1276 +#define _ITER_NEXT_RANGE_r03 1277 +#define _ITER_NEXT_RANGE_r13 1278 +#define _ITER_NEXT_RANGE_r23 1279 +#define _ITER_NEXT_TUPLE_r03 1280 +#define _ITER_NEXT_TUPLE_r13 1281 +#define _ITER_NEXT_TUPLE_r23 1282 +#define _JUMP_BACKWARD_NO_INTERRUPT_r00 1283 +#define _JUMP_BACKWARD_NO_INTERRUPT_r11 1284 +#define _JUMP_BACKWARD_NO_INTERRUPT_r22 1285 +#define _JUMP_BACKWARD_NO_INTERRUPT_r33 1286 +#define _JUMP_TO_TOP_r00 1287 +#define _LIST_APPEND_r10 1288 +#define _LIST_EXTEND_r11 1289 +#define _LOAD_ATTR_r10 1290 +#define _LOAD_ATTR_CLASS_r11 1291 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11 1292 +#define _LOAD_ATTR_INSTANCE_VALUE_r02 1293 +#define _LOAD_ATTR_INSTANCE_VALUE_r12 1294 +#define _LOAD_ATTR_INSTANCE_VALUE_r23 1295 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 1296 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 1297 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 1298 +#define _LOAD_ATTR_METHOD_NO_DICT_r02 1299 +#define _LOAD_ATTR_METHOD_NO_DICT_r12 1300 +#define _LOAD_ATTR_METHOD_NO_DICT_r23 1301 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1302 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1303 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1304 +#define _LOAD_ATTR_MODULE_r12 1305 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1306 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1307 +#define _LOAD_ATTR_PROPERTY_FRAME_r01 1308 +#define _LOAD_ATTR_PROPERTY_FRAME_r11 1309 +#define _LOAD_ATTR_PROPERTY_FRAME_r22 1310 +#define _LOAD_ATTR_PROPERTY_FRAME_r33 1311 +#define _LOAD_ATTR_SLOT_r02 1312 +#define _LOAD_ATTR_SLOT_r12 1313 +#define _LOAD_ATTR_SLOT_r23 1314 +#define _LOAD_ATTR_WITH_HINT_r12 1315 +#define _LOAD_BUILD_CLASS_r01 1316 +#define _LOAD_BYTECODE_r00 1317 +#define _LOAD_COMMON_CONSTANT_r01 1318 +#define _LOAD_COMMON_CONSTANT_r12 1319 +#define _LOAD_COMMON_CONSTANT_r23 1320 +#define _LOAD_CONST_r01 1321 +#define _LOAD_CONST_r12 1322 +#define _LOAD_CONST_r23 1323 +#define _LOAD_CONST_INLINE_r01 1324 +#define _LOAD_CONST_INLINE_r12 1325 +#define _LOAD_CONST_INLINE_r23 1326 +#define _LOAD_CONST_INLINE_BORROW_r01 1327 +#define _LOAD_CONST_INLINE_BORROW_r12 1328 +#define _LOAD_CONST_INLINE_BORROW_r23 1329 +#define _LOAD_DEREF_r01 1330 +#define _LOAD_FAST_r01 1331 +#define _LOAD_FAST_r12 1332 +#define _LOAD_FAST_r23 1333 +#define _LOAD_FAST_0_r01 1334 +#define _LOAD_FAST_0_r12 1335 +#define _LOAD_FAST_0_r23 1336 +#define _LOAD_FAST_1_r01 1337 +#define _LOAD_FAST_1_r12 1338 +#define _LOAD_FAST_1_r23 1339 +#define _LOAD_FAST_2_r01 1340 +#define _LOAD_FAST_2_r12 1341 +#define _LOAD_FAST_2_r23 1342 +#define _LOAD_FAST_3_r01 1343 +#define _LOAD_FAST_3_r12 1344 +#define _LOAD_FAST_3_r23 1345 +#define _LOAD_FAST_4_r01 1346 +#define _LOAD_FAST_4_r12 1347 +#define _LOAD_FAST_4_r23 1348 +#define _LOAD_FAST_5_r01 1349 +#define _LOAD_FAST_5_r12 1350 +#define _LOAD_FAST_5_r23 1351 +#define _LOAD_FAST_6_r01 1352 +#define _LOAD_FAST_6_r12 1353 +#define _LOAD_FAST_6_r23 1354 +#define _LOAD_FAST_7_r01 1355 +#define _LOAD_FAST_7_r12 1356 +#define _LOAD_FAST_7_r23 1357 +#define _LOAD_FAST_AND_CLEAR_r01 1358 +#define _LOAD_FAST_AND_CLEAR_r12 1359 +#define _LOAD_FAST_AND_CLEAR_r23 1360 +#define _LOAD_FAST_BORROW_r01 1361 +#define _LOAD_FAST_BORROW_r12 1362 +#define _LOAD_FAST_BORROW_r23 1363 +#define _LOAD_FAST_BORROW_0_r01 1364 +#define _LOAD_FAST_BORROW_0_r12 1365 +#define _LOAD_FAST_BORROW_0_r23 1366 +#define _LOAD_FAST_BORROW_1_r01 1367 +#define _LOAD_FAST_BORROW_1_r12 1368 +#define _LOAD_FAST_BORROW_1_r23 1369 +#define _LOAD_FAST_BORROW_2_r01 1370 +#define _LOAD_FAST_BORROW_2_r12 1371 +#define _LOAD_FAST_BORROW_2_r23 1372 +#define _LOAD_FAST_BORROW_3_r01 1373 +#define _LOAD_FAST_BORROW_3_r12 1374 +#define _LOAD_FAST_BORROW_3_r23 1375 +#define _LOAD_FAST_BORROW_4_r01 1376 +#define _LOAD_FAST_BORROW_4_r12 1377 +#define _LOAD_FAST_BORROW_4_r23 1378 +#define _LOAD_FAST_BORROW_5_r01 1379 +#define _LOAD_FAST_BORROW_5_r12 1380 +#define _LOAD_FAST_BORROW_5_r23 1381 +#define _LOAD_FAST_BORROW_6_r01 1382 +#define _LOAD_FAST_BORROW_6_r12 1383 +#define _LOAD_FAST_BORROW_6_r23 1384 +#define _LOAD_FAST_BORROW_7_r01 1385 +#define _LOAD_FAST_BORROW_7_r12 1386 +#define _LOAD_FAST_BORROW_7_r23 1387 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1388 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1389 +#define _LOAD_FAST_CHECK_r01 1390 +#define _LOAD_FAST_CHECK_r12 1391 +#define _LOAD_FAST_CHECK_r23 1392 +#define _LOAD_FAST_LOAD_FAST_r02 1393 +#define _LOAD_FAST_LOAD_FAST_r13 1394 +#define _LOAD_FROM_DICT_OR_DEREF_r11 1395 +#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1396 +#define _LOAD_GLOBAL_r00 1397 +#define _LOAD_GLOBAL_BUILTINS_r01 1398 +#define _LOAD_GLOBAL_MODULE_r01 1399 +#define _LOAD_LOCALS_r01 1400 +#define _LOAD_LOCALS_r12 1401 +#define _LOAD_LOCALS_r23 1402 +#define _LOAD_NAME_r01 1403 +#define _LOAD_SMALL_INT_r01 1404 +#define _LOAD_SMALL_INT_r12 1405 +#define _LOAD_SMALL_INT_r23 1406 +#define _LOAD_SMALL_INT_0_r01 1407 +#define _LOAD_SMALL_INT_0_r12 1408 +#define _LOAD_SMALL_INT_0_r23 1409 +#define _LOAD_SMALL_INT_1_r01 1410 +#define _LOAD_SMALL_INT_1_r12 1411 +#define _LOAD_SMALL_INT_1_r23 1412 +#define _LOAD_SMALL_INT_2_r01 1413 +#define _LOAD_SMALL_INT_2_r12 1414 +#define _LOAD_SMALL_INT_2_r23 1415 +#define _LOAD_SMALL_INT_3_r01 1416 +#define _LOAD_SMALL_INT_3_r12 1417 +#define _LOAD_SMALL_INT_3_r23 1418 +#define _LOAD_SPECIAL_r00 1419 +#define _LOAD_SUPER_ATTR_ATTR_r31 1420 +#define _LOAD_SUPER_ATTR_METHOD_r32 1421 +#define _LOCK_OBJECT_r01 1422 +#define _LOCK_OBJECT_r11 1423 +#define _LOCK_OBJECT_r22 1424 +#define _LOCK_OBJECT_r33 1425 +#define _MAKE_CALLARGS_A_TUPLE_r33 1426 +#define _MAKE_CELL_r00 1427 +#define _MAKE_FUNCTION_r12 1428 +#define _MAKE_HEAP_SAFE_r01 1429 +#define _MAKE_HEAP_SAFE_r11 1430 +#define _MAKE_HEAP_SAFE_r22 1431 +#define _MAKE_HEAP_SAFE_r33 1432 +#define _MAKE_WARM_r00 1433 +#define _MAKE_WARM_r11 1434 +#define _MAKE_WARM_r22 1435 +#define _MAKE_WARM_r33 1436 +#define _MAP_ADD_r20 1437 +#define _MATCH_CLASS_r33 1438 +#define _MATCH_KEYS_r23 1439 +#define _MATCH_MAPPING_r02 1440 +#define _MATCH_MAPPING_r12 1441 +#define _MATCH_MAPPING_r23 1442 +#define _MATCH_SEQUENCE_r02 1443 +#define _MATCH_SEQUENCE_r12 1444 +#define _MATCH_SEQUENCE_r23 1445 +#define _MAYBE_EXPAND_METHOD_r00 1446 +#define _MAYBE_EXPAND_METHOD_KW_r11 1447 +#define _MONITOR_CALL_r00 1448 +#define _MONITOR_CALL_KW_r11 1449 +#define _MONITOR_JUMP_BACKWARD_r00 1450 +#define _MONITOR_JUMP_BACKWARD_r11 1451 +#define _MONITOR_JUMP_BACKWARD_r22 1452 +#define _MONITOR_JUMP_BACKWARD_r33 1453 +#define _MONITOR_RESUME_r00 1454 +#define _NOP_r00 1455 +#define _NOP_r11 1456 +#define _NOP_r22 1457 +#define _NOP_r33 1458 +#define _POP_EXCEPT_r10 1459 +#define _POP_ITER_r20 1460 +#define _POP_JUMP_IF_FALSE_r00 1461 +#define _POP_JUMP_IF_FALSE_r10 1462 +#define _POP_JUMP_IF_FALSE_r21 1463 +#define _POP_JUMP_IF_FALSE_r32 1464 +#define _POP_JUMP_IF_TRUE_r00 1465 +#define _POP_JUMP_IF_TRUE_r10 1466 +#define _POP_JUMP_IF_TRUE_r21 1467 +#define _POP_JUMP_IF_TRUE_r32 1468 +#define _POP_TOP_r10 1469 +#define _POP_TOP_FLOAT_r00 1470 +#define _POP_TOP_FLOAT_r10 1471 +#define _POP_TOP_FLOAT_r21 1472 +#define _POP_TOP_FLOAT_r32 1473 +#define _POP_TOP_INT_r00 1474 +#define _POP_TOP_INT_r10 1475 +#define _POP_TOP_INT_r21 1476 +#define _POP_TOP_INT_r32 1477 +#define _POP_TOP_NOP_r00 1478 +#define _POP_TOP_NOP_r10 1479 +#define _POP_TOP_NOP_r21 1480 +#define _POP_TOP_NOP_r32 1481 +#define _POP_TOP_OPARG_r00 1482 +#define _POP_TOP_UNICODE_r00 1483 +#define _POP_TOP_UNICODE_r10 1484 +#define _POP_TOP_UNICODE_r21 1485 +#define _POP_TOP_UNICODE_r32 1486 +#define _PUSH_EXC_INFO_r02 1487 +#define _PUSH_EXC_INFO_r12 1488 +#define _PUSH_EXC_INFO_r23 1489 +#define _PUSH_FRAME_r10 1490 +#define _PUSH_NULL_r01 1491 +#define _PUSH_NULL_r12 1492 +#define _PUSH_NULL_r23 1493 +#define _PUSH_NULL_CONDITIONAL_r00 1494 +#define _PUSH_TAGGED_ZERO_r01 1495 +#define _PUSH_TAGGED_ZERO_r12 1496 +#define _PUSH_TAGGED_ZERO_r23 1497 +#define _PY_FRAME_EX_r31 1498 +#define _PY_FRAME_GENERAL_r01 1499 +#define _PY_FRAME_KW_r11 1500 +#define _REPLACE_WITH_TRUE_r02 1501 +#define _REPLACE_WITH_TRUE_r12 1502 +#define _REPLACE_WITH_TRUE_r23 1503 +#define _RESUME_CHECK_r00 1504 +#define _RESUME_CHECK_r11 1505 +#define _RESUME_CHECK_r22 1506 +#define _RESUME_CHECK_r33 1507 +#define _RETURN_GENERATOR_r01 1508 +#define _RETURN_VALUE_r11 1509 +#define _RROT_3_r03 1510 +#define _RROT_3_r13 1511 +#define _RROT_3_r23 1512 +#define _RROT_3_r33 1513 +#define _SAVE_RETURN_OFFSET_r00 1514 +#define _SAVE_RETURN_OFFSET_r11 1515 +#define _SAVE_RETURN_OFFSET_r22 1516 +#define _SAVE_RETURN_OFFSET_r33 1517 +#define _SEND_ASYNC_GEN_r33 1518 +#define _SEND_ASYNC_GEN_TIER_TWO_r33 1519 +#define _SEND_GEN_FRAME_r33 1520 +#define _SEND_VIRTUAL_r33 1521 +#define _SEND_VIRTUAL_TIER_TWO_r03 1522 +#define _SEND_VIRTUAL_TIER_TWO_r13 1523 +#define _SEND_VIRTUAL_TIER_TWO_r23 1524 +#define _SEND_VIRTUAL_TIER_TWO_r33 1525 +#define _SETUP_ANNOTATIONS_r00 1526 +#define _SET_ADD_r10 1527 +#define _SET_FUNCTION_ATTRIBUTE_r01 1528 +#define _SET_FUNCTION_ATTRIBUTE_r11 1529 +#define _SET_FUNCTION_ATTRIBUTE_r21 1530 +#define _SET_FUNCTION_ATTRIBUTE_r32 1531 +#define _SET_IP_r00 1532 +#define _SET_IP_r11 1533 +#define _SET_IP_r22 1534 +#define _SET_IP_r33 1535 +#define _SET_UPDATE_r11 1536 +#define _SPILL_OR_RELOAD_r01 1537 +#define _SPILL_OR_RELOAD_r02 1538 +#define _SPILL_OR_RELOAD_r03 1539 +#define _SPILL_OR_RELOAD_r10 1540 +#define _SPILL_OR_RELOAD_r12 1541 +#define _SPILL_OR_RELOAD_r13 1542 +#define _SPILL_OR_RELOAD_r20 1543 +#define _SPILL_OR_RELOAD_r21 1544 +#define _SPILL_OR_RELOAD_r23 1545 +#define _SPILL_OR_RELOAD_r30 1546 +#define _SPILL_OR_RELOAD_r31 1547 +#define _SPILL_OR_RELOAD_r32 1548 +#define _START_EXECUTOR_r00 1549 +#define _STORE_ATTR_r20 1550 +#define _STORE_ATTR_INSTANCE_VALUE_r21 1551 +#define _STORE_ATTR_SLOT_r21 1552 +#define _STORE_ATTR_WITH_HINT_r21 1553 +#define _STORE_DEREF_r10 1554 +#define _STORE_FAST_LOAD_FAST_r11 1555 +#define _STORE_FAST_STORE_FAST_r20 1556 +#define _STORE_GLOBAL_r10 1557 +#define _STORE_NAME_r10 1558 +#define _STORE_SLICE_r30 1559 +#define _STORE_SUBSCR_r30 1560 +#define _STORE_SUBSCR_DICT_r31 1561 +#define _STORE_SUBSCR_DICT_KNOWN_HASH_r31 1562 +#define _STORE_SUBSCR_LIST_INT_r32 1563 +#define _SWAP_r11 1564 +#define _SWAP_2_r02 1565 +#define _SWAP_2_r12 1566 +#define _SWAP_2_r22 1567 +#define _SWAP_2_r33 1568 +#define _SWAP_3_r03 1569 +#define _SWAP_3_r13 1570 +#define _SWAP_3_r23 1571 +#define _SWAP_3_r33 1572 +#define _SWAP_FAST_r01 1573 +#define _SWAP_FAST_r11 1574 +#define _SWAP_FAST_r22 1575 +#define _SWAP_FAST_r33 1576 +#define _SWAP_FAST_0_r01 1577 +#define _SWAP_FAST_0_r11 1578 +#define _SWAP_FAST_0_r22 1579 +#define _SWAP_FAST_0_r33 1580 +#define _SWAP_FAST_1_r01 1581 +#define _SWAP_FAST_1_r11 1582 +#define _SWAP_FAST_1_r22 1583 +#define _SWAP_FAST_1_r33 1584 +#define _SWAP_FAST_2_r01 1585 +#define _SWAP_FAST_2_r11 1586 +#define _SWAP_FAST_2_r22 1587 +#define _SWAP_FAST_2_r33 1588 +#define _SWAP_FAST_3_r01 1589 +#define _SWAP_FAST_3_r11 1590 +#define _SWAP_FAST_3_r22 1591 +#define _SWAP_FAST_3_r33 1592 +#define _SWAP_FAST_4_r01 1593 +#define _SWAP_FAST_4_r11 1594 +#define _SWAP_FAST_4_r22 1595 +#define _SWAP_FAST_4_r33 1596 +#define _SWAP_FAST_5_r01 1597 +#define _SWAP_FAST_5_r11 1598 +#define _SWAP_FAST_5_r22 1599 +#define _SWAP_FAST_5_r33 1600 +#define _SWAP_FAST_6_r01 1601 +#define _SWAP_FAST_6_r11 1602 +#define _SWAP_FAST_6_r22 1603 +#define _SWAP_FAST_6_r33 1604 +#define _SWAP_FAST_7_r01 1605 +#define _SWAP_FAST_7_r11 1606 +#define _SWAP_FAST_7_r22 1607 +#define _SWAP_FAST_7_r33 1608 +#define _TIER2_RESUME_CHECK_r00 1609 +#define _TIER2_RESUME_CHECK_r11 1610 +#define _TIER2_RESUME_CHECK_r22 1611 +#define _TIER2_RESUME_CHECK_r33 1612 +#define _TO_BOOL_r11 1613 +#define _TO_BOOL_BOOL_r01 1614 +#define _TO_BOOL_BOOL_r11 1615 +#define _TO_BOOL_BOOL_r22 1616 +#define _TO_BOOL_BOOL_r33 1617 +#define _TO_BOOL_INT_r02 1618 +#define _TO_BOOL_INT_r12 1619 +#define _TO_BOOL_INT_r23 1620 +#define _TO_BOOL_LIST_r02 1621 +#define _TO_BOOL_LIST_r12 1622 +#define _TO_BOOL_LIST_r23 1623 +#define _TO_BOOL_NONE_r01 1624 +#define _TO_BOOL_NONE_r11 1625 +#define _TO_BOOL_NONE_r22 1626 +#define _TO_BOOL_NONE_r33 1627 +#define _TO_BOOL_STR_r02 1628 +#define _TO_BOOL_STR_r12 1629 +#define _TO_BOOL_STR_r23 1630 +#define _TRACE_RECORD_r00 1631 +#define _UNARY_INVERT_r12 1632 +#define _UNARY_NEGATIVE_r12 1633 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r02 1634 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r12 1635 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r23 1636 +#define _UNARY_NOT_r01 1637 +#define _UNARY_NOT_r11 1638 +#define _UNARY_NOT_r22 1639 +#define _UNARY_NOT_r33 1640 +#define _UNPACK_EX_r10 1641 +#define _UNPACK_SEQUENCE_r10 1642 +#define _UNPACK_SEQUENCE_LIST_r10 1643 +#define _UNPACK_SEQUENCE_TUPLE_r10 1644 +#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1645 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 1646 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 1647 +#define _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 1648 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 1649 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 1650 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 1651 +#define _WITH_EXCEPT_START_r33 1652 +#define _YIELD_VALUE_r11 1653 +#define MAX_UOP_REGS_ID 1653 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index ff7e800aa9bb1a8..43498c57d8a9d1e 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -11,20 +11,35 @@ extern "C" { #include #include "pycore_uop_ids.h" -extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; +#define MAX_CACHED_REGISTER 3 +extern const uint32_t _PyUop_Flags[MAX_UOP_ID+1]; typedef struct _rep_range { uint8_t start; uint8_t stop; } ReplicationRange; extern const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1]; -extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; +extern const char * const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1]; extern int _PyUop_num_popped(int opcode, int oparg); +typedef struct _pyuop_tos_cache_entry { + /* input depth is implicit in position */ + int8_t output; + int8_t exit; + int16_t opcode; +} _PyUopTOSentry; +typedef struct _PyUopCachingInfo { + uint8_t best[MAX_CACHED_REGISTER + 1]; + _PyUopTOSentry entries[MAX_CACHED_REGISTER + 1]; +} _PyUopCachingInfo; +extern const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1]; +extern const uint16_t _PyUop_SpillsAndReloads[4][4]; +extern const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1]; + #ifdef NEED_OPCODE_METADATA -const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { +const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_NOP] = HAS_PURE_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, - [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, @@ -44,50 +59,48 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_FAST_BORROW_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_BORROW] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_LOAD_FAST_BORROW_LOAD_FAST_BORROW] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG, [_LOAD_SMALL_INT_0] = 0, [_LOAD_SMALL_INT_1] = 0, [_LOAD_SMALL_INT_2] = 0, [_LOAD_SMALL_INT_3] = 0, [_LOAD_SMALL_INT] = HAS_ARG_FLAG, - [_STORE_FAST_0] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_1] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_2] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_3] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_4] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_5] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_6] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_7] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_SWAP_FAST_0] = HAS_LOCAL_FLAG, + [_SWAP_FAST_1] = HAS_LOCAL_FLAG, + [_SWAP_FAST_2] = HAS_LOCAL_FLAG, + [_SWAP_FAST_3] = HAS_LOCAL_FLAG, + [_SWAP_FAST_4] = HAS_LOCAL_FLAG, + [_SWAP_FAST_5] = HAS_LOCAL_FLAG, + [_SWAP_FAST_6] = HAS_LOCAL_FLAG, + [_SWAP_FAST_7] = HAS_LOCAL_FLAG, + [_SWAP_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_POP_TOP] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_POP_TOP_NOP] = 0, [_POP_TOP_INT] = 0, [_POP_TOP_FLOAT] = 0, [_POP_TOP_UNICODE] = 0, - [_POP_TWO] = HAS_ESCAPES_FLAG, + [_POP_TOP_OPARG] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_PUSH_NULL] = HAS_PURE_FLAG, [_END_FOR] = HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG, [_POP_ITER] = HAS_ESCAPES_FLAG, [_END_SEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_UNARY_NOT] = HAS_PURE_FLAG, + [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_UNARY_NEGATIVE_FLOAT_INPLACE] = 0, + [_UNARY_NOT] = 0, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_TO_BOOL_BOOL] = HAS_EXIT_FLAG, - [_TO_BOOL_INT] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_TO_BOOL_INT] = 0, [_GUARD_NOS_LIST] = HAS_EXIT_FLAG, [_GUARD_TOS_LIST] = HAS_EXIT_FLAG, [_GUARD_TOS_SLICE] = HAS_EXIT_FLAG, - [_TO_BOOL_LIST] = HAS_ESCAPES_FLAG, + [_TO_BOOL_LIST] = 0, [_TO_BOOL_NONE] = HAS_EXIT_FLAG, + [_GUARD_NOS_COMPACT_ASCII] = HAS_EXIT_FLAG, [_GUARD_NOS_UNICODE] = HAS_EXIT_FLAG, [_GUARD_TOS_UNICODE] = HAS_EXIT_FLAG, - [_TO_BOOL_STR] = HAS_ESCAPES_FLAG, - [_REPLACE_WITH_TRUE] = HAS_ESCAPES_FLAG, - [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_TO_BOOL_STR] = 0, + [_REPLACE_WITH_TRUE] = 0, + [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_INT] = HAS_EXIT_FLAG, [_GUARD_TOS_INT] = HAS_EXIT_FLAG, [_GUARD_NOS_OVERFLOWED] = HAS_EXIT_FLAG, @@ -95,60 +108,90 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_MULTIPLY_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_INT_INPLACE] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBTRACT_INT_INPLACE] = HAS_EXIT_FLAG, + [_BINARY_OP_MULTIPLY_INT_INPLACE] = HAS_EXIT_FLAG, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG, [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, - [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_BINARY_OP_EXTEND] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_EXTEND] = HAS_ESCAPES_FLAG, + [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT_INPLACE] = 0, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE] = 0, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE] = 0, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT] = 0, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT] = 0, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT] = 0, + [_BINARY_OP_TRUEDIV_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_BINARY_OP_EXTEND_LHS] = HAS_EXIT_FLAG, + [_GUARD_BINARY_OP_EXTEND_RHS] = HAS_EXIT_FLAG, + [_GUARD_BINARY_OP_EXTEND] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_EXTEND] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_SUBSCR_LIST_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_SUBSCR_STR_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_LIST_INT] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_LIST_SLICE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_STR_INT] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBSCR_USTR_INT] = HAS_EXIT_FLAG, [_GUARD_NOS_TUPLE] = HAS_EXIT_FLAG, [_GUARD_TOS_TUPLE] = HAS_EXIT_FLAG, - [_BINARY_OP_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_NOS_DICT] = HAS_EXIT_FLAG, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBSCR_TUPLE_INT] = 0, + [_GUARD_NOS_DICT_SUBSCRIPT] = HAS_DEOPT_FLAG, + [_GUARD_NOS_DICT_STORE_SUBSCRIPT] = HAS_DEOPT_FLAG, + [_GUARD_TOS_ANY_DICT] = HAS_EXIT_FLAG, [_GUARD_TOS_DICT] = HAS_EXIT_FLAG, - [_BINARY_OP_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_SUBSCR_CHECK_FUNC] = HAS_DEOPT_FLAG, + [_GUARD_TOS_FROZENDICT] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_CHECK_FUNC] = HAS_EXIT_FLAG, [_BINARY_OP_SUBSCR_INIT_CALL] = 0, [_LIST_APPEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SUBSCR_DICT_KNOWN_HASH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_RETURN_VALUE] = HAS_ESCAPES_FLAG, + [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_HEAP_SAFE] = 0, + [_RETURN_VALUE] = HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_YIELD_VALUE] = HAS_ARG_FLAG, + [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_GUARD_TOS_IS_NONE] = HAS_EXIT_FLAG, + [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, + [_SEND_VIRTUAL_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_GUARD_3OS_ASYNC_GEN_ASEND] = HAS_EXIT_FLAG, + [_SEND_ASYNC_GEN_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE] = 0, + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE] = 0, + [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE_UNIQUE_TUPLE] = HAS_ARG_FLAG, [_UNPACK_SEQUENCE_LIST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_EX] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_LOCALS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_LOCALS] = HAS_ERROR_FLAG, [_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_NULL_CONDITIONAL] = HAS_ARG_FLAG, @@ -167,53 +210,68 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BUILD_TEMPLATE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_TUPLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, [_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_BUILD_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SETUP_ANNOTATIONS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DICT_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DICT_MERGE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DICT_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_DICT_MERGE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAP_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_NOS_TYPE_VERSION] = HAS_EXIT_FLAG, + [_GUARD_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG, - [_GUARD_TYPE_VERSION_AND_LOCK] = HAS_EXIT_FLAG, - [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG, - [_LOAD_ATTR_INSTANCE_VALUE] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_SLOT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_TYPE_VERSION_LOCKED] = HAS_EXIT_FLAG, + [_GUARD_TYPE] = HAS_EXIT_FLAG, + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_EXIT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_MODULE] = HAS_EXIT_FLAG, + [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG, + [_LOAD_ATTR_SLOT] = HAS_EXIT_FLAG, [_CHECK_ATTR_CLASS] = HAS_EXIT_FLAG, [_LOAD_ATTR_CLASS] = HAS_ESCAPES_FLAG, - [_LOAD_ATTR_PROPERTY_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_ATTR_PROPERTY_FRAME] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG, [_GUARD_DORV_NO_DICT] = HAS_EXIT_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = HAS_ESCAPES_FLAG, + [_LOCK_OBJECT] = HAS_DEOPT_FLAG, [_STORE_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_STORE_ATTR_SLOT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG, [_COMPARE_OP_INT] = HAS_ARG_FLAG, [_COMPARE_OP_STR] = HAS_ARG_FLAG, - [_IS_OP] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_TOS_ANY_SET] = HAS_DEOPT_FLAG, - [_CONTAINS_OP_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CONTAINS_OP_DICT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_IS_OP] = HAS_ARG_FLAG, + [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_TOS_ANY_SET] = HAS_EXIT_FLAG, + [_GUARD_TOS_SET] = HAS_EXIT_FLAG, + [_GUARD_TOS_FROZENSET] = HAS_EXIT_FLAG, + [_CONTAINS_OP_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CONTAINS_OP_DICT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EG_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EXC_MATCH] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_IMPORT_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_IMPORT_FROM] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_IS_NONE] = HAS_ESCAPES_FLAG, [_GET_LEN] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_MATCH_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MATCH_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MATCH_MAPPING] = 0, [_MATCH_SEQUENCE] = 0, [_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GET_ITER] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_ITERATOR] = HAS_EXIT_FLAG, + [_GUARD_ITER_VIRTUAL] = HAS_EXIT_FLAG, + [_PUSH_TAGGED_ZERO] = 0, + [_GET_ITER_TRAD] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_TYPE_ITER] = HAS_EXIT_FLAG, + [_ITER_NEXT_INLINE] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_NOS_ITER_VIRTUAL] = HAS_EXIT_FLAG, + [_GUARD_TOS_NOT_NULL] = HAS_EXIT_FLAG, + [_FOR_ITER_VIRTUAL_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, [_ITER_NEXT_LIST_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, @@ -223,21 +281,20 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_ITER_CHECK_RANGE] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_EXIT_FLAG, [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG, - [_FOR_ITER_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_FOR_ITER_GEN_FRAME] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_INSERT_NULL] = 0, [_LOAD_SPECIAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_EXC_INFO] = 0, - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG, - [_GUARD_KEYS_VERSION] = HAS_DEOPT_FLAG, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_EXIT_FLAG, [_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG, [_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, + [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_EXIT_FLAG, [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, [_MAYBE_EXPAND_METHOD] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_FUNCTION_VERSION_INLINE] = HAS_EXIT_FLAG, [_CHECK_METHOD_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -256,44 +313,61 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_PUSH_FRAME] = 0, - [_GUARD_NOS_NULL] = HAS_DEOPT_FLAG, - [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, - [_GUARD_THIRD_NULL] = HAS_DEOPT_FLAG, - [_GUARD_CALLABLE_TYPE_1] = HAS_DEOPT_FLAG, - [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_CALLABLE_STR_1] = HAS_DEOPT_FLAG, - [_CALL_STR_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_CALLABLE_TUPLE_1] = HAS_DEOPT_FLAG, - [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CHECK_AND_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_PUSH_FRAME] = HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_GUARD_NOS_NULL] = HAS_EXIT_FLAG, + [_GUARD_THIRD_NULL] = HAS_EXIT_FLAG, + [_GUARD_CALLABLE_TYPE_1] = HAS_EXIT_FLAG, + [_CALL_TYPE_1] = HAS_ARG_FLAG, + [_GUARD_CALLABLE_STR_1] = HAS_EXIT_FLAG, + [_CALL_STR_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_TUPLE_1] = HAS_EXIT_FLAG, + [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_OBJECT] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_CALLABLE_LEN] = HAS_DEOPT_FLAG, + [_GUARD_CALLABLE_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_BUILTIN_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_LEN] = HAS_EXIT_FLAG, [_CALL_LEN] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_CALLABLE_ISINSTANCE] = HAS_DEOPT_FLAG, + [_GUARD_CALLABLE_ISINSTANCE] = HAS_EXIT_FLAG, [_CALL_ISINSTANCE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_CALLABLE_LIST_APPEND] = HAS_DEOPT_FLAG, - [_CALL_LIST_APPEND] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_LIST_APPEND] = HAS_EXIT_FLAG, + [_CALL_LIST_APPEND] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_RECURSION_LIMIT] = HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_O_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAYBE_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_METHOD_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_CHECK_IS_NOT_PY_CALLABLE_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CALL_KW_NON_PY] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_CALLARGS_A_TUPLE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_IS_PY_CALLABLE_EX] = HAS_EXIT_FLAG, + [_PY_FRAME_EX] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, + [_CHECK_IS_NOT_PY_CALLABLE_EX] = HAS_EXIT_FLAG, + [_CALL_FUNCTION_EX_NON_PY_GENERAL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG, - [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -308,363 +382,5825 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, - [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_BIT_IS_SET_POP_4] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_SET_POP_5] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_SET_POP_6] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_SET_POP_7] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_SET_POP] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_GUARD_BIT_IS_UNSET_POP_4] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_UNSET_POP_5] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_UNSET_POP_6] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_UNSET_POP_7] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_UNSET_POP] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, [_JUMP_TO_TOP] = 0, [_SET_IP] = 0, [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, [_EXIT_TRACE] = HAS_ESCAPES_FLAG, + [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, - [_POP_TOP_LOAD_CONST_INLINE] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, - [_POP_CALL] = HAS_ESCAPES_FLAG, - [_POP_CALL_ONE] = HAS_ESCAPES_FLAG, - [_POP_CALL_TWO] = HAS_ESCAPES_FLAG, - [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_POP_TWO_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_POP_CALL_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_LOAD_CONST_UNDER_INLINE] = 0, - [_LOAD_CONST_UNDER_INLINE_BORROW] = 0, - [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, - [_START_EXECUTOR] = 0, + [_RROT_3] = HAS_PURE_FLAG, + [_START_EXECUTOR] = HAS_DEOPT_FLAG, [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, - [_DEOPT] = 0, - [_ERROR_POP_N] = HAS_ARG_FLAG, - [_TIER2_RESUME_CHECK] = HAS_DEOPT_FLAG, + [_DEOPT] = HAS_SYNC_SP_FLAG, + [_HANDLE_PENDING_AND_DEOPT] = HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, + [_ERROR_POP_N] = HAS_ARG_FLAG | HAS_SYNC_SP_FLAG, + [_SPILL_OR_RELOAD] = 0, + [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, + [_COLD_EXIT] = HAS_SYNC_SP_FLAG, + [_COLD_DYNAMIC_EXIT] = HAS_SYNC_SP_FLAG, + [_GUARD_CODE_VERSION__PUSH_FRAME] = HAS_EXIT_FLAG, + [_GUARD_CODE_VERSION_YIELD_VALUE] = HAS_EXIT_FLAG, + [_GUARD_CODE_VERSION_RETURN_VALUE] = HAS_EXIT_FLAG, + [_GUARD_CODE_VERSION_RETURN_GENERATOR] = HAS_EXIT_FLAG, + [_GUARD_IP__PUSH_FRAME] = HAS_EXIT_FLAG, + [_GUARD_IP_YIELD_VALUE] = HAS_EXIT_FLAG, + [_GUARD_IP_RETURN_VALUE] = HAS_EXIT_FLAG, + [_GUARD_IP_RETURN_GENERATOR] = HAS_EXIT_FLAG, + [_RECORD_TOS] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_TOS_TYPE] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_NOS] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_NOS_TYPE] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_NOS_GEN_FUNC] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_3OS_GEN_FUNC] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_4OS] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_CALLABLE] = HAS_ARG_FLAG | HAS_RECORDS_VALUE_FLAG, + [_RECORD_CALLABLE_KW] = HAS_ARG_FLAG | HAS_RECORDS_VALUE_FLAG, + [_RECORD_BOUND_METHOD] = HAS_ARG_FLAG | HAS_RECORDS_VALUE_FLAG, + [_RECORD_CODE] = HAS_RECORDS_VALUE_FLAG, }; const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { [_LOAD_FAST] = { 0, 8 }, [_LOAD_FAST_BORROW] = { 0, 8 }, [_LOAD_SMALL_INT] = { 0, 4 }, - [_STORE_FAST] = { 0, 8 }, + [_SWAP_FAST] = { 0, 8 }, [_INIT_CALL_PY_EXACT_ARGS] = { 0, 5 }, [_COPY] = { 1, 4 }, [_SWAP] = { 2, 4 }, + [_GUARD_BIT_IS_SET_POP] = { 4, 8 }, + [_GUARD_BIT_IS_UNSET_POP] = { 4, 8 }, +}; + +const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { + [_NOP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _NOP_r00 }, + { 1, 1, _NOP_r11 }, + { 2, 2, _NOP_r22 }, + { 3, 3, _NOP_r33 }, + }, + }, + [_CHECK_PERIODIC] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_PERIODIC_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_RESUME_CHECK] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _RESUME_CHECK_r00 }, + { 1, 1, _RESUME_CHECK_r11 }, + { 2, 2, _RESUME_CHECK_r22 }, + { 3, 3, _RESUME_CHECK_r33 }, + }, + }, + [_LOAD_FAST_CHECK] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_CHECK_r01 }, + { 2, 1, _LOAD_FAST_CHECK_r12 }, + { 3, 2, _LOAD_FAST_CHECK_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_0] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_0_r01 }, + { 2, 1, _LOAD_FAST_0_r12 }, + { 3, 2, _LOAD_FAST_0_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_1] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_1_r01 }, + { 2, 1, _LOAD_FAST_1_r12 }, + { 3, 2, _LOAD_FAST_1_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_2] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_2_r01 }, + { 2, 1, _LOAD_FAST_2_r12 }, + { 3, 2, _LOAD_FAST_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_3] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_3_r01 }, + { 2, 1, _LOAD_FAST_3_r12 }, + { 3, 2, _LOAD_FAST_3_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_4] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_4_r01 }, + { 2, 1, _LOAD_FAST_4_r12 }, + { 3, 2, _LOAD_FAST_4_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_5] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_5_r01 }, + { 2, 1, _LOAD_FAST_5_r12 }, + { 3, 2, _LOAD_FAST_5_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_6] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_6_r01 }, + { 2, 1, _LOAD_FAST_6_r12 }, + { 3, 2, _LOAD_FAST_6_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_7] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_7_r01 }, + { 2, 1, _LOAD_FAST_7_r12 }, + { 3, 2, _LOAD_FAST_7_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_r01 }, + { 2, 1, _LOAD_FAST_r12 }, + { 3, 2, _LOAD_FAST_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_0] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_0_r01 }, + { 2, 1, _LOAD_FAST_BORROW_0_r12 }, + { 3, 2, _LOAD_FAST_BORROW_0_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_1] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_1_r01 }, + { 2, 1, _LOAD_FAST_BORROW_1_r12 }, + { 3, 2, _LOAD_FAST_BORROW_1_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_2] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_2_r01 }, + { 2, 1, _LOAD_FAST_BORROW_2_r12 }, + { 3, 2, _LOAD_FAST_BORROW_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_3] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_3_r01 }, + { 2, 1, _LOAD_FAST_BORROW_3_r12 }, + { 3, 2, _LOAD_FAST_BORROW_3_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_4] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_4_r01 }, + { 2, 1, _LOAD_FAST_BORROW_4_r12 }, + { 3, 2, _LOAD_FAST_BORROW_4_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_5] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_5_r01 }, + { 2, 1, _LOAD_FAST_BORROW_5_r12 }, + { 3, 2, _LOAD_FAST_BORROW_5_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_6] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_6_r01 }, + { 2, 1, _LOAD_FAST_BORROW_6_r12 }, + { 3, 2, _LOAD_FAST_BORROW_6_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_7] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_7_r01 }, + { 2, 1, _LOAD_FAST_BORROW_7_r12 }, + { 3, 2, _LOAD_FAST_BORROW_7_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_r01 }, + { 2, 1, _LOAD_FAST_BORROW_r12 }, + { 3, 2, _LOAD_FAST_BORROW_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_AND_CLEAR] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_AND_CLEAR_r01 }, + { 2, 1, _LOAD_FAST_AND_CLEAR_r12 }, + { 3, 2, _LOAD_FAST_AND_CLEAR_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_CONST] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_CONST_r01 }, + { 2, 1, _LOAD_CONST_r12 }, + { 3, 2, _LOAD_CONST_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT_0] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_0_r01 }, + { 2, 1, _LOAD_SMALL_INT_0_r12 }, + { 3, 2, _LOAD_SMALL_INT_0_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT_1] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_1_r01 }, + { 2, 1, _LOAD_SMALL_INT_1_r12 }, + { 3, 2, _LOAD_SMALL_INT_1_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT_2] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_2_r01 }, + { 2, 1, _LOAD_SMALL_INT_2_r12 }, + { 3, 2, _LOAD_SMALL_INT_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT_3] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_3_r01 }, + { 2, 1, _LOAD_SMALL_INT_3_r12 }, + { 3, 2, _LOAD_SMALL_INT_3_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_r01 }, + { 2, 1, _LOAD_SMALL_INT_r12 }, + { 3, 2, _LOAD_SMALL_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_SWAP_FAST_0] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_0_r01 }, + { 1, 1, _SWAP_FAST_0_r11 }, + { 2, 2, _SWAP_FAST_0_r22 }, + { 3, 3, _SWAP_FAST_0_r33 }, + }, + }, + [_SWAP_FAST_1] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_1_r01 }, + { 1, 1, _SWAP_FAST_1_r11 }, + { 2, 2, _SWAP_FAST_1_r22 }, + { 3, 3, _SWAP_FAST_1_r33 }, + }, + }, + [_SWAP_FAST_2] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_2_r01 }, + { 1, 1, _SWAP_FAST_2_r11 }, + { 2, 2, _SWAP_FAST_2_r22 }, + { 3, 3, _SWAP_FAST_2_r33 }, + }, + }, + [_SWAP_FAST_3] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_3_r01 }, + { 1, 1, _SWAP_FAST_3_r11 }, + { 2, 2, _SWAP_FAST_3_r22 }, + { 3, 3, _SWAP_FAST_3_r33 }, + }, + }, + [_SWAP_FAST_4] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_4_r01 }, + { 1, 1, _SWAP_FAST_4_r11 }, + { 2, 2, _SWAP_FAST_4_r22 }, + { 3, 3, _SWAP_FAST_4_r33 }, + }, + }, + [_SWAP_FAST_5] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_5_r01 }, + { 1, 1, _SWAP_FAST_5_r11 }, + { 2, 2, _SWAP_FAST_5_r22 }, + { 3, 3, _SWAP_FAST_5_r33 }, + }, + }, + [_SWAP_FAST_6] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_6_r01 }, + { 1, 1, _SWAP_FAST_6_r11 }, + { 2, 2, _SWAP_FAST_6_r22 }, + { 3, 3, _SWAP_FAST_6_r33 }, + }, + }, + [_SWAP_FAST_7] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_7_r01 }, + { 1, 1, _SWAP_FAST_7_r11 }, + { 2, 2, _SWAP_FAST_7_r22 }, + { 3, 3, _SWAP_FAST_7_r33 }, + }, + }, + [_SWAP_FAST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_r01 }, + { 1, 1, _SWAP_FAST_r11 }, + { 2, 2, _SWAP_FAST_r22 }, + { 3, 3, _SWAP_FAST_r33 }, + }, + }, + [_POP_TOP] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _POP_TOP_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_POP_TOP_NOP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _POP_TOP_NOP_r00 }, + { 0, 1, _POP_TOP_NOP_r10 }, + { 1, 2, _POP_TOP_NOP_r21 }, + { 2, 3, _POP_TOP_NOP_r32 }, + }, + }, + [_POP_TOP_INT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _POP_TOP_INT_r00 }, + { 0, 1, _POP_TOP_INT_r10 }, + { 1, 2, _POP_TOP_INT_r21 }, + { 2, 3, _POP_TOP_INT_r32 }, + }, + }, + [_POP_TOP_FLOAT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _POP_TOP_FLOAT_r00 }, + { 0, 1, _POP_TOP_FLOAT_r10 }, + { 1, 2, _POP_TOP_FLOAT_r21 }, + { 2, 3, _POP_TOP_FLOAT_r32 }, + }, + }, + [_POP_TOP_UNICODE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _POP_TOP_UNICODE_r00 }, + { 0, 1, _POP_TOP_UNICODE_r10 }, + { 1, 2, _POP_TOP_UNICODE_r21 }, + { 2, 3, _POP_TOP_UNICODE_r32 }, + }, + }, + [_POP_TOP_OPARG] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _POP_TOP_OPARG_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PUSH_NULL] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _PUSH_NULL_r01 }, + { 2, 1, _PUSH_NULL_r12 }, + { 3, 2, _PUSH_NULL_r23 }, + { -1, -1, -1 }, + }, + }, + [_END_FOR] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _END_FOR_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_POP_ITER] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 2, _POP_ITER_r20 }, + { -1, -1, -1 }, + }, + }, + [_END_SEND] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _END_SEND_r31 }, + }, + }, + [_UNARY_NEGATIVE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _UNARY_NEGATIVE_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNARY_NEGATIVE_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _UNARY_NEGATIVE_FLOAT_INPLACE_r02 }, + { 2, 1, _UNARY_NEGATIVE_FLOAT_INPLACE_r12 }, + { 3, 2, _UNARY_NEGATIVE_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_UNARY_NOT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _UNARY_NOT_r01 }, + { 1, 1, _UNARY_NOT_r11 }, + { 2, 2, _UNARY_NOT_r22 }, + { 3, 3, _UNARY_NOT_r33 }, + }, + }, + [_TO_BOOL] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _TO_BOOL_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_TO_BOOL_BOOL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _TO_BOOL_BOOL_r01 }, + { 1, 1, _TO_BOOL_BOOL_r11 }, + { 2, 2, _TO_BOOL_BOOL_r22 }, + { 3, 3, _TO_BOOL_BOOL_r33 }, + }, + }, + [_TO_BOOL_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _TO_BOOL_INT_r02 }, + { 2, 1, _TO_BOOL_INT_r12 }, + { 3, 2, _TO_BOOL_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_LIST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_LIST_r02 }, + { 2, 1, _GUARD_NOS_LIST_r12 }, + { 2, 2, _GUARD_NOS_LIST_r22 }, + { 3, 3, _GUARD_NOS_LIST_r33 }, + }, + }, + [_GUARD_TOS_LIST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_LIST_r01 }, + { 1, 1, _GUARD_TOS_LIST_r11 }, + { 2, 2, _GUARD_TOS_LIST_r22 }, + { 3, 3, _GUARD_TOS_LIST_r33 }, + }, + }, + [_GUARD_TOS_SLICE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_SLICE_r01 }, + { 1, 1, _GUARD_TOS_SLICE_r11 }, + { 2, 2, _GUARD_TOS_SLICE_r22 }, + { 3, 3, _GUARD_TOS_SLICE_r33 }, + }, + }, + [_TO_BOOL_LIST] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _TO_BOOL_LIST_r02 }, + { 2, 1, _TO_BOOL_LIST_r12 }, + { 3, 2, _TO_BOOL_LIST_r23 }, + { -1, -1, -1 }, + }, + }, + [_TO_BOOL_NONE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _TO_BOOL_NONE_r01 }, + { 1, 1, _TO_BOOL_NONE_r11 }, + { 2, 2, _TO_BOOL_NONE_r22 }, + { 3, 3, _TO_BOOL_NONE_r33 }, + }, + }, + [_GUARD_NOS_COMPACT_ASCII] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_COMPACT_ASCII_r02 }, + { 2, 1, _GUARD_NOS_COMPACT_ASCII_r12 }, + { 2, 2, _GUARD_NOS_COMPACT_ASCII_r22 }, + { 3, 3, _GUARD_NOS_COMPACT_ASCII_r33 }, + }, + }, + [_GUARD_NOS_UNICODE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_UNICODE_r02 }, + { 2, 1, _GUARD_NOS_UNICODE_r12 }, + { 2, 2, _GUARD_NOS_UNICODE_r22 }, + { 3, 3, _GUARD_NOS_UNICODE_r33 }, + }, + }, + [_GUARD_TOS_UNICODE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_UNICODE_r01 }, + { 1, 1, _GUARD_TOS_UNICODE_r11 }, + { 2, 2, _GUARD_TOS_UNICODE_r22 }, + { 3, 3, _GUARD_TOS_UNICODE_r33 }, + }, + }, + [_TO_BOOL_STR] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _TO_BOOL_STR_r02 }, + { 2, 1, _TO_BOOL_STR_r12 }, + { 3, 2, _TO_BOOL_STR_r23 }, + { -1, -1, -1 }, + }, + }, + [_REPLACE_WITH_TRUE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _REPLACE_WITH_TRUE_r02 }, + { 2, 1, _REPLACE_WITH_TRUE_r12 }, + { 3, 2, _REPLACE_WITH_TRUE_r23 }, + { -1, -1, -1 }, + }, + }, + [_UNARY_INVERT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _UNARY_INVERT_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_INT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_INT_r02 }, + { 2, 1, _GUARD_NOS_INT_r12 }, + { 2, 2, _GUARD_NOS_INT_r22 }, + { 3, 3, _GUARD_NOS_INT_r33 }, + }, + }, + [_GUARD_TOS_INT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_INT_r01 }, + { 1, 1, _GUARD_TOS_INT_r11 }, + { 2, 2, _GUARD_TOS_INT_r22 }, + { 3, 3, _GUARD_TOS_INT_r33 }, + }, + }, + [_GUARD_NOS_OVERFLOWED] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_OVERFLOWED_r02 }, + { 2, 1, _GUARD_NOS_OVERFLOWED_r12 }, + { 2, 2, _GUARD_NOS_OVERFLOWED_r22 }, + { 3, 3, _GUARD_NOS_OVERFLOWED_r33 }, + }, + }, + [_GUARD_TOS_OVERFLOWED] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_OVERFLOWED_r01 }, + { 1, 1, _GUARD_TOS_OVERFLOWED_r11 }, + { 2, 2, _GUARD_TOS_OVERFLOWED_r22 }, + { 3, 3, _GUARD_TOS_OVERFLOWED_r33 }, + }, + }, + [_BINARY_OP_MULTIPLY_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_INT_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_INT_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_INT_r03 }, + { 3, 1, _BINARY_OP_ADD_INT_r13 }, + { 3, 2, _BINARY_OP_ADD_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_INT_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_INT_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_INT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_INT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_ADD_INT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_ADD_INT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_INT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_INT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_INT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_INT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_MULTIPLY_INT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_INT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_INT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_INT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_INT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_ADD_INT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_ADD_INT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_FLOAT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_FLOAT_r02 }, + { 2, 1, _GUARD_NOS_FLOAT_r12 }, + { 2, 2, _GUARD_NOS_FLOAT_r22 }, + { 3, 3, _GUARD_NOS_FLOAT_r33 }, + }, + }, + [_GUARD_TOS_FLOAT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_FLOAT_r01 }, + { 1, 1, _GUARD_TOS_FLOAT_r11 }, + { 2, 2, _GUARD_TOS_FLOAT_r22 }, + { 3, 3, _GUARD_TOS_FLOAT_r33 }, + }, + }, + [_BINARY_OP_MULTIPLY_FLOAT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_FLOAT_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_FLOAT_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_FLOAT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_FLOAT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_FLOAT_r03 }, + { 3, 1, _BINARY_OP_ADD_FLOAT_r13 }, + { 3, 2, _BINARY_OP_ADD_FLOAT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_FLOAT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_FLOAT_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_FLOAT_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_FLOAT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_FLOAT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_ADD_FLOAT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_ADD_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_TRUEDIV_FLOAT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_TRUEDIV_FLOAT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_UNICODE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_UNICODE_r03 }, + { 3, 1, _BINARY_OP_ADD_UNICODE_r13 }, + { 3, 2, _BINARY_OP_ADD_UNICODE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_INPLACE_ADD_UNICODE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BINARY_OP_INPLACE_ADD_UNICODE_r21 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_BINARY_OP_EXTEND_LHS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_BINARY_OP_EXTEND_LHS_r02 }, + { 2, 1, _GUARD_BINARY_OP_EXTEND_LHS_r12 }, + { 2, 2, _GUARD_BINARY_OP_EXTEND_LHS_r22 }, + { 3, 3, _GUARD_BINARY_OP_EXTEND_LHS_r33 }, + }, + }, + [_GUARD_BINARY_OP_EXTEND_RHS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_BINARY_OP_EXTEND_RHS_r02 }, + { 2, 1, _GUARD_BINARY_OP_EXTEND_RHS_r12 }, + { 2, 2, _GUARD_BINARY_OP_EXTEND_RHS_r22 }, + { 3, 3, _GUARD_BINARY_OP_EXTEND_RHS_r33 }, + }, + }, + [_GUARD_BINARY_OP_EXTEND] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 2, _GUARD_BINARY_OP_EXTEND_r22 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_EXTEND] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_EXTEND_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_SLICE] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _BINARY_SLICE_r31 }, + }, + }, + [_STORE_SLICE] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 3, _STORE_SLICE_r30 }, + }, + }, + [_BINARY_OP_SUBSCR_LIST_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_LIST_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_LIST_SLICE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_LIST_SLICE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_STR_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_STR_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_USTR_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_USTR_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_TUPLE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_TUPLE_r02 }, + { 2, 1, _GUARD_NOS_TUPLE_r12 }, + { 2, 2, _GUARD_NOS_TUPLE_r22 }, + { 3, 3, _GUARD_NOS_TUPLE_r33 }, + }, + }, + [_GUARD_TOS_TUPLE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_TUPLE_r01 }, + { 1, 1, _GUARD_TOS_TUPLE_r11 }, + { 2, 2, _GUARD_TOS_TUPLE_r22 }, + { 3, 3, _GUARD_TOS_TUPLE_r33 }, + }, + }, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02 }, + { 2, 1, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12 }, + { 2, 2, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22 }, + { 3, 3, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33 }, + }, + }, + [_BINARY_OP_SUBSCR_TUPLE_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBSCR_TUPLE_INT_r03 }, + { 3, 1, _BINARY_OP_SUBSCR_TUPLE_INT_r13 }, + { 3, 2, _BINARY_OP_SUBSCR_TUPLE_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_DICT_SUBSCRIPT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_DICT_SUBSCRIPT_r02 }, + { 2, 1, _GUARD_NOS_DICT_SUBSCRIPT_r12 }, + { 2, 2, _GUARD_NOS_DICT_SUBSCRIPT_r22 }, + { 3, 3, _GUARD_NOS_DICT_SUBSCRIPT_r33 }, + }, + }, + [_GUARD_NOS_DICT_STORE_SUBSCRIPT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_NOS_DICT_STORE_SUBSCRIPT_r03 }, + { 3, 1, _GUARD_NOS_DICT_STORE_SUBSCRIPT_r13 }, + { 3, 2, _GUARD_NOS_DICT_STORE_SUBSCRIPT_r23 }, + { 3, 3, _GUARD_NOS_DICT_STORE_SUBSCRIPT_r33 }, + }, + }, + [_GUARD_TOS_ANY_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_ANY_DICT_r01 }, + { 1, 1, _GUARD_TOS_ANY_DICT_r11 }, + { 2, 2, _GUARD_TOS_ANY_DICT_r22 }, + { 3, 3, _GUARD_TOS_ANY_DICT_r33 }, + }, + }, + [_GUARD_TOS_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_DICT_r01 }, + { 1, 1, _GUARD_TOS_DICT_r11 }, + { 2, 2, _GUARD_TOS_DICT_r22 }, + { 3, 3, _GUARD_TOS_DICT_r33 }, + }, + }, + [_GUARD_TOS_FROZENDICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_FROZENDICT_r01 }, + { 1, 1, _GUARD_TOS_FROZENDICT_r11 }, + { 2, 2, _GUARD_TOS_FROZENDICT_r22 }, + { 3, 3, _GUARD_TOS_FROZENDICT_r33 }, + }, + }, + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_DICT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_DICT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_CHECK_FUNC] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_CHECK_FUNC_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_INIT_CALL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _BINARY_OP_SUBSCR_INIT_CALL_r01 }, + { 1, 1, _BINARY_OP_SUBSCR_INIT_CALL_r11 }, + { 1, 2, _BINARY_OP_SUBSCR_INIT_CALL_r21 }, + { 1, 3, _BINARY_OP_SUBSCR_INIT_CALL_r31 }, + }, + }, + [_LIST_APPEND] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _LIST_APPEND_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SET_ADD] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _SET_ADD_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_SUBSCR] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 3, _STORE_SUBSCR_r30 }, + }, + }, + [_STORE_SUBSCR_LIST_INT] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 3, _STORE_SUBSCR_LIST_INT_r32 }, + }, + }, + [_STORE_SUBSCR_DICT] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _STORE_SUBSCR_DICT_r31 }, + }, + }, + [_STORE_SUBSCR_DICT_KNOWN_HASH] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _STORE_SUBSCR_DICT_KNOWN_HASH_r31 }, + }, + }, + [_DELETE_SUBSCR] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 2, _DELETE_SUBSCR_r20 }, + { -1, -1, -1 }, + }, + }, + [_CALL_INTRINSIC_1] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _CALL_INTRINSIC_1_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_INTRINSIC_2] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _CALL_INTRINSIC_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_MAKE_HEAP_SAFE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _MAKE_HEAP_SAFE_r01 }, + { 1, 1, _MAKE_HEAP_SAFE_r11 }, + { 2, 2, _MAKE_HEAP_SAFE_r22 }, + { 3, 3, _MAKE_HEAP_SAFE_r33 }, + }, + }, + [_RETURN_VALUE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _RETURN_VALUE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GET_AITER] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _GET_AITER_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GET_ANEXT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _GET_ANEXT_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GET_AWAITABLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _GET_AWAITABLE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SEND_GEN_FRAME] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _SEND_GEN_FRAME_r33 }, + }, + }, + [_GUARD_TOS_IS_NONE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_IS_NONE_r01 }, + { 1, 1, _GUARD_TOS_IS_NONE_r11 }, + { 2, 2, _GUARD_TOS_IS_NONE_r22 }, + { 3, 3, _GUARD_TOS_IS_NONE_r33 }, + }, + }, + [_GUARD_NOS_NOT_NULL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_NOT_NULL_r02 }, + { 2, 1, _GUARD_NOS_NOT_NULL_r12 }, + { 2, 2, _GUARD_NOS_NOT_NULL_r22 }, + { 3, 3, _GUARD_NOS_NOT_NULL_r33 }, + }, + }, + [_SEND_VIRTUAL_TIER_TWO] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _SEND_VIRTUAL_TIER_TWO_r03 }, + { 3, 1, _SEND_VIRTUAL_TIER_TWO_r13 }, + { 3, 2, _SEND_VIRTUAL_TIER_TWO_r23 }, + { 3, 3, _SEND_VIRTUAL_TIER_TWO_r33 }, + }, + }, + [_GUARD_3OS_ASYNC_GEN_ASEND] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_3OS_ASYNC_GEN_ASEND_r03 }, + { 3, 1, _GUARD_3OS_ASYNC_GEN_ASEND_r13 }, + { 3, 2, _GUARD_3OS_ASYNC_GEN_ASEND_r23 }, + { 3, 3, _GUARD_3OS_ASYNC_GEN_ASEND_r33 }, + }, + }, + [_SEND_ASYNC_GEN_TIER_TWO] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _SEND_ASYNC_GEN_TIER_TWO_r33 }, + }, + }, + [_YIELD_VALUE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _YIELD_VALUE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_POP_EXCEPT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _POP_EXCEPT_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_COMMON_CONSTANT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_COMMON_CONSTANT_r01 }, + { 2, 1, _LOAD_COMMON_CONSTANT_r12 }, + { 3, 2, _LOAD_COMMON_CONSTANT_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_BUILD_CLASS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_BUILD_CLASS_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_NAME] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _STORE_NAME_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DELETE_NAME] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _DELETE_NAME_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_SEQUENCE_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_TWO_TUPLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _UNPACK_SEQUENCE_TWO_TUPLE_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 }, + { 2, 1, _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 }, + { 3, 2, _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE] = { + .best = { 0, 1, 1, 1 }, + .entries = { + { 3, 0, _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 }, + { 3, 1, _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_TUPLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_SEQUENCE_TUPLE_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_UNIQUE_TUPLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_LIST] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_SEQUENCE_LIST_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_EX] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_EX_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_ATTR] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 2, _STORE_ATTR_r20 }, + { -1, -1, -1 }, + }, + }, + [_DELETE_ATTR] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _DELETE_ATTR_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_GLOBAL] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _STORE_GLOBAL_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DELETE_GLOBAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _DELETE_GLOBAL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_LOCALS] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_LOCALS_r01 }, + { 2, 1, _LOAD_LOCALS_r12 }, + { 3, 2, _LOAD_LOCALS_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_NAME] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_NAME_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_GLOBAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _LOAD_GLOBAL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PUSH_NULL_CONDITIONAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _PUSH_NULL_CONDITIONAL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_GLOBALS_VERSION] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_GLOBALS_VERSION_r00 }, + { 1, 1, _GUARD_GLOBALS_VERSION_r11 }, + { 2, 2, _GUARD_GLOBALS_VERSION_r22 }, + { 3, 3, _GUARD_GLOBALS_VERSION_r33 }, + }, + }, + [_LOAD_GLOBAL_MODULE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_GLOBAL_MODULE_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_GLOBAL_BUILTINS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_GLOBAL_BUILTINS_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DELETE_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _DELETE_FAST_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAKE_CELL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _MAKE_CELL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DELETE_DEREF] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _DELETE_DEREF_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FROM_DICT_OR_DEREF] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_FROM_DICT_OR_DEREF_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_DEREF] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_DEREF_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_DEREF] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _STORE_DEREF_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_COPY_FREE_VARS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _COPY_FREE_VARS_r00 }, + { 1, 1, _COPY_FREE_VARS_r11 }, + { 2, 2, _COPY_FREE_VARS_r22 }, + { 3, 3, _COPY_FREE_VARS_r33 }, + }, + }, + [_BUILD_STRING] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_STRING_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_INTERPOLATION] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_INTERPOLATION_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_TEMPLATE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BUILD_TEMPLATE_r21 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_TUPLE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_TUPLE_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_LIST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_LIST_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LIST_EXTEND] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LIST_EXTEND_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SET_UPDATE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _SET_UPDATE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_SET] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_SET_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_MAP] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_MAP_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SETUP_ANNOTATIONS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _SETUP_ANNOTATIONS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DICT_UPDATE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _DICT_UPDATE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DICT_MERGE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _DICT_MERGE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAP_ADD] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 2, _MAP_ADD_r20 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SUPER_ATTR_ATTR] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _LOAD_SUPER_ATTR_ATTR_r31 }, + }, + }, + [_GUARD_NOS_TYPE_VERSION] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_TYPE_VERSION_r02 }, + { 2, 1, _GUARD_NOS_TYPE_VERSION_r12 }, + { 2, 2, _GUARD_NOS_TYPE_VERSION_r22 }, + { 3, 3, _GUARD_NOS_TYPE_VERSION_r33 }, + }, + }, + [_GUARD_LOAD_SUPER_ATTR_METHOD] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_LOAD_SUPER_ATTR_METHOD_r03 }, + { 3, 1, _GUARD_LOAD_SUPER_ATTR_METHOD_r13 }, + { 3, 2, _GUARD_LOAD_SUPER_ATTR_METHOD_r23 }, + { 3, 3, _GUARD_LOAD_SUPER_ATTR_METHOD_r33 }, + }, + }, + [_LOAD_SUPER_ATTR_METHOD] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 3, _LOAD_SUPER_ATTR_METHOD_r32 }, + }, + }, + [_LOAD_ATTR] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _LOAD_ATTR_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_TYPE_VERSION] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TYPE_VERSION_r01 }, + { 1, 1, _GUARD_TYPE_VERSION_r11 }, + { 2, 2, _GUARD_TYPE_VERSION_r22 }, + { 3, 3, _GUARD_TYPE_VERSION_r33 }, + }, + }, + [_GUARD_TYPE_VERSION_LOCKED] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TYPE_VERSION_LOCKED_r01 }, + { 1, 1, _GUARD_TYPE_VERSION_LOCKED_r11 }, + { 2, 2, _GUARD_TYPE_VERSION_LOCKED_r22 }, + { 3, 3, _GUARD_TYPE_VERSION_LOCKED_r33 }, + }, + }, + [_GUARD_TYPE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TYPE_r01 }, + { 1, 1, _GUARD_TYPE_r11 }, + { 2, 2, _GUARD_TYPE_r22 }, + { 3, 3, _GUARD_TYPE_r33 }, + }, + }, + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _CHECK_MANAGED_OBJECT_HAS_VALUES_r01 }, + { 1, 1, _CHECK_MANAGED_OBJECT_HAS_VALUES_r11 }, + { 2, 2, _CHECK_MANAGED_OBJECT_HAS_VALUES_r22 }, + { 3, 3, _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 }, + }, + }, + [_LOAD_ATTR_INSTANCE_VALUE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _LOAD_ATTR_INSTANCE_VALUE_r02 }, + { 2, 1, _LOAD_ATTR_INSTANCE_VALUE_r12 }, + { 3, 2, _LOAD_ATTR_INSTANCE_VALUE_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_MODULE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _LOAD_ATTR_MODULE_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_WITH_HINT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _LOAD_ATTR_WITH_HINT_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_SLOT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _LOAD_ATTR_SLOT_r02 }, + { 2, 1, _LOAD_ATTR_SLOT_r12 }, + { 3, 2, _LOAD_ATTR_SLOT_r23 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_ATTR_CLASS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _CHECK_ATTR_CLASS_r01 }, + { 1, 1, _CHECK_ATTR_CLASS_r11 }, + { 2, 2, _CHECK_ATTR_CLASS_r22 }, + { 3, 3, _CHECK_ATTR_CLASS_r33 }, + }, + }, + [_LOAD_ATTR_CLASS] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_CLASS_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_PROPERTY_FRAME] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _LOAD_ATTR_PROPERTY_FRAME_r01 }, + { 1, 1, _LOAD_ATTR_PROPERTY_FRAME_r11 }, + { 2, 2, _LOAD_ATTR_PROPERTY_FRAME_r22 }, + { 3, 3, _LOAD_ATTR_PROPERTY_FRAME_r33 }, + }, + }, + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_DORV_NO_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_DORV_NO_DICT_r01 }, + { 1, 1, _GUARD_DORV_NO_DICT_r11 }, + { 2, 2, _GUARD_DORV_NO_DICT_r22 }, + { 3, 3, _GUARD_DORV_NO_DICT_r33 }, + }, + }, + [_STORE_ATTR_INSTANCE_VALUE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _STORE_ATTR_INSTANCE_VALUE_r21 }, + { -1, -1, -1 }, + }, + }, + [_LOCK_OBJECT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _LOCK_OBJECT_r01 }, + { 1, 1, _LOCK_OBJECT_r11 }, + { 2, 2, _LOCK_OBJECT_r22 }, + { 3, 3, _LOCK_OBJECT_r33 }, + }, + }, + [_STORE_ATTR_WITH_HINT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _STORE_ATTR_WITH_HINT_r21 }, + { -1, -1, -1 }, + }, + }, + [_STORE_ATTR_SLOT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _STORE_ATTR_SLOT_r21 }, + { -1, -1, -1 }, + }, + }, + [_COMPARE_OP] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _COMPARE_OP_r21 }, + { -1, -1, -1 }, + }, + }, + [_COMPARE_OP_FLOAT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _COMPARE_OP_FLOAT_r03 }, + { 3, 1, _COMPARE_OP_FLOAT_r13 }, + { 3, 2, _COMPARE_OP_FLOAT_r23 }, + { -1, -1, -1 }, + }, + }, + [_COMPARE_OP_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _COMPARE_OP_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_COMPARE_OP_STR] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _COMPARE_OP_STR_r23 }, + { -1, -1, -1 }, + }, + }, + [_IS_OP] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _IS_OP_r03 }, + { 3, 1, _IS_OP_r13 }, + { 3, 2, _IS_OP_r23 }, + { -1, -1, -1 }, + }, + }, + [_CONTAINS_OP] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _CONTAINS_OP_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_TOS_ANY_SET] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_ANY_SET_r01 }, + { 1, 1, _GUARD_TOS_ANY_SET_r11 }, + { 2, 2, _GUARD_TOS_ANY_SET_r22 }, + { 3, 3, _GUARD_TOS_ANY_SET_r33 }, + }, + }, + [_GUARD_TOS_SET] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_SET_r01 }, + { 1, 1, _GUARD_TOS_SET_r11 }, + { 2, 2, _GUARD_TOS_SET_r22 }, + { 3, 3, _GUARD_TOS_SET_r33 }, + }, + }, + [_GUARD_TOS_FROZENSET] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_FROZENSET_r01 }, + { 1, 1, _GUARD_TOS_FROZENSET_r11 }, + { 2, 2, _GUARD_TOS_FROZENSET_r22 }, + { 3, 3, _GUARD_TOS_FROZENSET_r33 }, + }, + }, + [_CONTAINS_OP_SET] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _CONTAINS_OP_SET_r23 }, + { -1, -1, -1 }, + }, + }, + [_CONTAINS_OP_DICT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _CONTAINS_OP_DICT_r23 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_EG_MATCH] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 2, _CHECK_EG_MATCH_r22 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_EXC_MATCH] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 2, _CHECK_EXC_MATCH_r22 }, + { -1, -1, -1 }, + }, + }, + [_IMPORT_NAME] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _IMPORT_NAME_r21 }, + { -1, -1, -1 }, + }, + }, + [_IMPORT_FROM] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _IMPORT_FROM_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_IS_NONE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _IS_NONE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GET_LEN] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _GET_LEN_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MATCH_CLASS] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _MATCH_CLASS_r33 }, + }, + }, + [_MATCH_MAPPING] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _MATCH_MAPPING_r02 }, + { 2, 1, _MATCH_MAPPING_r12 }, + { 3, 2, _MATCH_MAPPING_r23 }, + { -1, -1, -1 }, + }, + }, + [_MATCH_SEQUENCE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _MATCH_SEQUENCE_r02 }, + { 2, 1, _MATCH_SEQUENCE_r12 }, + { 3, 2, _MATCH_SEQUENCE_r23 }, + { -1, -1, -1 }, + }, + }, + [_MATCH_KEYS] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _MATCH_KEYS_r23 }, + { -1, -1, -1 }, + }, + }, + [_GET_ITER] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _GET_ITER_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_ITERATOR] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_ITERATOR_r01 }, + { 1, 1, _GUARD_ITERATOR_r11 }, + { 2, 2, _GUARD_ITERATOR_r22 }, + { 3, 3, _GUARD_ITERATOR_r33 }, + }, + }, + [_GUARD_ITER_VIRTUAL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_ITER_VIRTUAL_r01 }, + { 1, 1, _GUARD_ITER_VIRTUAL_r11 }, + { 2, 2, _GUARD_ITER_VIRTUAL_r22 }, + { 3, 3, _GUARD_ITER_VIRTUAL_r33 }, + }, + }, + [_PUSH_TAGGED_ZERO] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _PUSH_TAGGED_ZERO_r01 }, + { 2, 1, _PUSH_TAGGED_ZERO_r12 }, + { 3, 2, _PUSH_TAGGED_ZERO_r23 }, + { -1, -1, -1 }, + }, + }, + [_GET_ITER_TRAD] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _GET_ITER_TRAD_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_FOR_ITER_TIER_TWO] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _FOR_ITER_TIER_TWO_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_TYPE_ITER] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_TYPE_ITER_r02 }, + { 2, 1, _GUARD_TYPE_ITER_r12 }, + { 2, 2, _GUARD_TYPE_ITER_r22 }, + { 3, 3, _GUARD_TYPE_ITER_r33 }, + }, + }, + [_ITER_NEXT_INLINE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _ITER_NEXT_INLINE_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_ITER_VIRTUAL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_ITER_VIRTUAL_r02 }, + { 2, 1, _GUARD_NOS_ITER_VIRTUAL_r12 }, + { 2, 2, _GUARD_NOS_ITER_VIRTUAL_r22 }, + { 3, 3, _GUARD_NOS_ITER_VIRTUAL_r33 }, + }, + }, + [_GUARD_TOS_NOT_NULL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_NOT_NULL_r01 }, + { 1, 1, _GUARD_TOS_NOT_NULL_r11 }, + { 2, 2, _GUARD_TOS_NOT_NULL_r22 }, + { 3, 3, _GUARD_TOS_NOT_NULL_r33 }, + }, + }, + [_FOR_ITER_VIRTUAL_TIER_TWO] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _FOR_ITER_VIRTUAL_TIER_TWO_r23 }, + { -1, -1, -1 }, + }, + }, + [_ITER_CHECK_LIST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _ITER_CHECK_LIST_r02 }, + { 2, 1, _ITER_CHECK_LIST_r12 }, + { 2, 2, _ITER_CHECK_LIST_r22 }, + { 3, 3, _ITER_CHECK_LIST_r33 }, + }, + }, + [_GUARD_NOT_EXHAUSTED_LIST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOT_EXHAUSTED_LIST_r02 }, + { 2, 1, _GUARD_NOT_EXHAUSTED_LIST_r12 }, + { 2, 2, _GUARD_NOT_EXHAUSTED_LIST_r22 }, + { 3, 3, _GUARD_NOT_EXHAUSTED_LIST_r33 }, + }, + }, + [_ITER_NEXT_LIST_TIER_TWO] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _ITER_NEXT_LIST_TIER_TWO_r23 }, + { -1, -1, -1 }, + }, + }, + [_ITER_CHECK_TUPLE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _ITER_CHECK_TUPLE_r02 }, + { 2, 1, _ITER_CHECK_TUPLE_r12 }, + { 2, 2, _ITER_CHECK_TUPLE_r22 }, + { 3, 3, _ITER_CHECK_TUPLE_r33 }, + }, + }, + [_GUARD_NOT_EXHAUSTED_TUPLE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOT_EXHAUSTED_TUPLE_r02 }, + { 2, 1, _GUARD_NOT_EXHAUSTED_TUPLE_r12 }, + { 2, 2, _GUARD_NOT_EXHAUSTED_TUPLE_r22 }, + { 3, 3, _GUARD_NOT_EXHAUSTED_TUPLE_r33 }, + }, + }, + [_ITER_NEXT_TUPLE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _ITER_NEXT_TUPLE_r03 }, + { 3, 1, _ITER_NEXT_TUPLE_r13 }, + { 3, 2, _ITER_NEXT_TUPLE_r23 }, + { -1, -1, -1 }, + }, + }, + [_ITER_CHECK_RANGE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _ITER_CHECK_RANGE_r02 }, + { 2, 1, _ITER_CHECK_RANGE_r12 }, + { 2, 2, _ITER_CHECK_RANGE_r22 }, + { 3, 3, _ITER_CHECK_RANGE_r33 }, + }, + }, + [_GUARD_NOT_EXHAUSTED_RANGE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOT_EXHAUSTED_RANGE_r02 }, + { 2, 1, _GUARD_NOT_EXHAUSTED_RANGE_r12 }, + { 2, 2, _GUARD_NOT_EXHAUSTED_RANGE_r22 }, + { 3, 3, _GUARD_NOT_EXHAUSTED_RANGE_r33 }, + }, + }, + [_ITER_NEXT_RANGE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _ITER_NEXT_RANGE_r03 }, + { 3, 1, _ITER_NEXT_RANGE_r13 }, + { 3, 2, _ITER_NEXT_RANGE_r23 }, + { -1, -1, -1 }, + }, + }, + [_FOR_ITER_GEN_FRAME] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _FOR_ITER_GEN_FRAME_r03 }, + { 3, 1, _FOR_ITER_GEN_FRAME_r13 }, + { 3, 2, _FOR_ITER_GEN_FRAME_r23 }, + { -1, -1, -1 }, + }, + }, + [_INSERT_NULL] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _INSERT_NULL_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SPECIAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _LOAD_SPECIAL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_WITH_EXCEPT_START] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _WITH_EXCEPT_START_r33 }, + }, + }, + [_PUSH_EXC_INFO] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _PUSH_EXC_INFO_r02 }, + { 2, 1, _PUSH_EXC_INFO_r12 }, + { 3, 2, _PUSH_EXC_INFO_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01 }, + { 1, 1, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 }, + { 2, 2, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 }, + { 3, 3, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 }, + }, + }, + [_LOAD_ATTR_METHOD_WITH_VALUES] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _LOAD_ATTR_METHOD_WITH_VALUES_r02 }, + { 2, 1, _LOAD_ATTR_METHOD_WITH_VALUES_r12 }, + { 3, 2, _LOAD_ATTR_METHOD_WITH_VALUES_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_METHOD_NO_DICT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _LOAD_ATTR_METHOD_NO_DICT_r02 }, + { 2, 1, _LOAD_ATTR_METHOD_NO_DICT_r12 }, + { 3, 2, _LOAD_ATTR_METHOD_NO_DICT_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_ATTR_METHOD_LAZY_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _CHECK_ATTR_METHOD_LAZY_DICT_r01 }, + { 1, 1, _CHECK_ATTR_METHOD_LAZY_DICT_r11 }, + { 2, 2, _CHECK_ATTR_METHOD_LAZY_DICT_r22 }, + { 3, 3, _CHECK_ATTR_METHOD_LAZY_DICT_r33 }, + }, + }, + [_LOAD_ATTR_METHOD_LAZY_DICT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _LOAD_ATTR_METHOD_LAZY_DICT_r02 }, + { 2, 1, _LOAD_ATTR_METHOD_LAZY_DICT_r12 }, + { 3, 2, _LOAD_ATTR_METHOD_LAZY_DICT_r23 }, + { -1, -1, -1 }, + }, + }, + [_MAYBE_EXPAND_METHOD] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _MAYBE_EXPAND_METHOD_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PY_FRAME_GENERAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 1, _PY_FRAME_GENERAL_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_FUNCTION_VERSION] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_FUNCTION_VERSION_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_FUNCTION_VERSION_INLINE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_FUNCTION_VERSION_INLINE_r00 }, + { 1, 1, _CHECK_FUNCTION_VERSION_INLINE_r11 }, + { 2, 2, _CHECK_FUNCTION_VERSION_INLINE_r22 }, + { 3, 3, _CHECK_FUNCTION_VERSION_INLINE_r33 }, + }, + }, + [_CHECK_METHOD_VERSION] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_METHOD_VERSION_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_EXPAND_METHOD] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _EXPAND_METHOD_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_IS_NOT_PY_CALLABLE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_IS_NOT_PY_CALLABLE_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_NON_PY_GENERAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _CALL_NON_PY_GENERAL_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_PEP_523] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_PEP_523_r00 }, + { 1, 1, _CHECK_PEP_523_r11 }, + { 2, 2, _CHECK_PEP_523_r22 }, + { 3, 3, _CHECK_PEP_523_r33 }, + }, + }, + [_CHECK_FUNCTION_EXACT_ARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_FUNCTION_EXACT_ARGS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_STACK_SPACE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_STACK_SPACE_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_RECURSION_REMAINING] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_RECURSION_REMAINING_r00 }, + { 1, 1, _CHECK_RECURSION_REMAINING_r11 }, + { 2, 2, _CHECK_RECURSION_REMAINING_r22 }, + { 3, 3, _CHECK_RECURSION_REMAINING_r33 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_0] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_0_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_1] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_1_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_2] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_2_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_3] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_3_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_4] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_4_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PUSH_FRAME] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _PUSH_FRAME_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_NULL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_NULL_r02 }, + { 2, 1, _GUARD_NOS_NULL_r12 }, + { 2, 2, _GUARD_NOS_NULL_r22 }, + { 3, 3, _GUARD_NOS_NULL_r33 }, + }, + }, + [_GUARD_THIRD_NULL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_THIRD_NULL_r03 }, + { 3, 1, _GUARD_THIRD_NULL_r13 }, + { 3, 2, _GUARD_THIRD_NULL_r23 }, + { 3, 3, _GUARD_THIRD_NULL_r33 }, + }, + }, + [_GUARD_CALLABLE_TYPE_1] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_CALLABLE_TYPE_1_r03 }, + { 3, 1, _GUARD_CALLABLE_TYPE_1_r13 }, + { 3, 2, _GUARD_CALLABLE_TYPE_1_r23 }, + { 3, 3, _GUARD_CALLABLE_TYPE_1_r33 }, + }, + }, + [_CALL_TYPE_1] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _CALL_TYPE_1_r02 }, + { 2, 1, _CALL_TYPE_1_r12 }, + { 2, 2, _CALL_TYPE_1_r22 }, + { 2, 3, _CALL_TYPE_1_r32 }, + }, + }, + [_GUARD_CALLABLE_STR_1] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_CALLABLE_STR_1_r03 }, + { 3, 1, _GUARD_CALLABLE_STR_1_r13 }, + { 3, 2, _GUARD_CALLABLE_STR_1_r23 }, + { 3, 3, _GUARD_CALLABLE_STR_1_r33 }, + }, + }, + [_CALL_STR_1] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 3, _CALL_STR_1_r32 }, + }, + }, + [_GUARD_CALLABLE_TUPLE_1] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_CALLABLE_TUPLE_1_r03 }, + { 3, 1, _GUARD_CALLABLE_TUPLE_1_r13 }, + { 3, 2, _GUARD_CALLABLE_TUPLE_1_r23 }, + { 3, 3, _GUARD_CALLABLE_TUPLE_1_r33 }, + }, + }, + [_CALL_TUPLE_1] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 3, _CALL_TUPLE_1_r32 }, + }, + }, + [_CHECK_OBJECT] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_OBJECT_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_ALLOCATE_OBJECT] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _ALLOCATE_OBJECT_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CREATE_INIT_FRAME] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 1, _CREATE_INIT_FRAME_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_EXIT_INIT_CHECK] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _EXIT_INIT_CHECK_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_BUILTIN_CLASS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_BUILTIN_CLASS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_BUILTIN_CLASS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_BUILTIN_CLASS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_BUILTIN_O] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_BUILTIN_O_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_BUILTIN_O] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 3, 0, _CALL_BUILTIN_O_r03 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_BUILTIN_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_BUILTIN_FAST_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_BUILTIN_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_BUILTIN_FAST_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_BUILTIN_FAST_WITH_KEYWORDS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_LEN] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_CALLABLE_LEN_r03 }, + { 3, 1, _GUARD_CALLABLE_LEN_r13 }, + { 3, 2, _GUARD_CALLABLE_LEN_r23 }, + { 3, 3, _GUARD_CALLABLE_LEN_r33 }, + }, + }, + [_CALL_LEN] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _CALL_LEN_r33 }, + }, + }, + [_GUARD_CALLABLE_ISINSTANCE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_CALLABLE_ISINSTANCE_r03 }, + { 3, 1, _GUARD_CALLABLE_ISINSTANCE_r13 }, + { 3, 2, _GUARD_CALLABLE_ISINSTANCE_r23 }, + { 3, 3, _GUARD_CALLABLE_ISINSTANCE_r33 }, + }, + }, + [_CALL_ISINSTANCE] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _CALL_ISINSTANCE_r31 }, + }, + }, + [_GUARD_CALLABLE_LIST_APPEND] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_CALLABLE_LIST_APPEND_r03 }, + { 3, 1, _GUARD_CALLABLE_LIST_APPEND_r13 }, + { 3, 2, _GUARD_CALLABLE_LIST_APPEND_r23 }, + { 3, 3, _GUARD_CALLABLE_LIST_APPEND_r33 }, + }, + }, + [_CALL_LIST_APPEND] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _CALL_LIST_APPEND_r03 }, + { 3, 1, _CALL_LIST_APPEND_r13 }, + { 3, 2, _CALL_LIST_APPEND_r23 }, + { 3, 3, _CALL_LIST_APPEND_r33 }, + }, + }, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_O] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 3, 0, _CALL_METHOD_DESCRIPTOR_O_r03 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_RECURSION_LIMIT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_RECURSION_LIMIT_r00 }, + { 1, 1, _CHECK_RECURSION_LIMIT_r11 }, + { 2, 2, _CHECK_RECURSION_LIMIT_r22 }, + { 3, 3, _CHECK_RECURSION_LIMIT_r33 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_O_INLINE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 3, 0, _CALL_METHOD_DESCRIPTOR_O_INLINE_r03 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_NOARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 3, 0, _CALL_METHOD_DESCRIPTOR_NOARGS_r03 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 3, 0, _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAYBE_EXPAND_METHOD_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _MAYBE_EXPAND_METHOD_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PY_FRAME_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _PY_FRAME_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_FUNCTION_VERSION_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _CHECK_FUNCTION_VERSION_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_METHOD_VERSION_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _CHECK_METHOD_VERSION_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_EXPAND_METHOD_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _EXPAND_METHOD_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_IS_NOT_PY_CALLABLE_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _CHECK_IS_NOT_PY_CALLABLE_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_KW_NON_PY] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _CALL_KW_NON_PY_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAKE_CALLARGS_A_TUPLE] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _MAKE_CALLARGS_A_TUPLE_r33 }, + }, + }, + [_CHECK_IS_PY_CALLABLE_EX] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _CHECK_IS_PY_CALLABLE_EX_r03 }, + { 3, 1, _CHECK_IS_PY_CALLABLE_EX_r13 }, + { 3, 2, _CHECK_IS_PY_CALLABLE_EX_r23 }, + { 3, 3, _CHECK_IS_PY_CALLABLE_EX_r33 }, + }, + }, + [_PY_FRAME_EX] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 1, _PY_FRAME_EX_r31 }, + }, + }, + [_CHECK_IS_NOT_PY_CALLABLE_EX] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _CHECK_IS_NOT_PY_CALLABLE_EX_r03 }, + { 3, 1, _CHECK_IS_NOT_PY_CALLABLE_EX_r13 }, + { 3, 2, _CHECK_IS_NOT_PY_CALLABLE_EX_r23 }, + { 3, 3, _CHECK_IS_NOT_PY_CALLABLE_EX_r33 }, + }, + }, + [_CALL_FUNCTION_EX_NON_PY_GENERAL] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _CALL_FUNCTION_EX_NON_PY_GENERAL_r31 }, + }, + }, + [_MAKE_FUNCTION] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _MAKE_FUNCTION_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SET_FUNCTION_ATTRIBUTE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SET_FUNCTION_ATTRIBUTE_r01 }, + { 1, 1, _SET_FUNCTION_ATTRIBUTE_r11 }, + { 1, 2, _SET_FUNCTION_ATTRIBUTE_r21 }, + { 2, 3, _SET_FUNCTION_ATTRIBUTE_r32 }, + }, + }, + [_RETURN_GENERATOR] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 1, _RETURN_GENERATOR_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_SLICE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_SLICE_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CONVERT_VALUE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _CONVERT_VALUE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_FORMAT_SIMPLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _FORMAT_SIMPLE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_FORMAT_WITH_SPEC] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _FORMAT_WITH_SPEC_r21 }, + { -1, -1, -1 }, + }, + }, + [_COPY_1] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _COPY_1_r02 }, + { 2, 1, _COPY_1_r12 }, + { 3, 2, _COPY_1_r23 }, + { -1, -1, -1 }, + }, + }, + [_COPY_2] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _COPY_2_r03 }, + { 3, 1, _COPY_2_r13 }, + { 3, 2, _COPY_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_COPY_3] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _COPY_3_r03 }, + { 3, 1, _COPY_3_r13 }, + { 3, 2, _COPY_3_r23 }, + { 3, 3, _COPY_3_r33 }, + }, + }, + [_COPY] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _COPY_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_r23 }, + { -1, -1, -1 }, + }, + }, + [_SWAP_2] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _SWAP_2_r02 }, + { 2, 1, _SWAP_2_r12 }, + { 2, 2, _SWAP_2_r22 }, + { 3, 3, _SWAP_2_r33 }, + }, + }, + [_SWAP_3] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _SWAP_3_r03 }, + { 3, 1, _SWAP_3_r13 }, + { 3, 2, _SWAP_3_r23 }, + { 3, 3, _SWAP_3_r33 }, + }, + }, + [_SWAP] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _SWAP_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_IS_TRUE_POP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IS_TRUE_POP_r00 }, + { 0, 0, _GUARD_IS_TRUE_POP_r10 }, + { 1, 1, _GUARD_IS_TRUE_POP_r21 }, + { 2, 2, _GUARD_IS_TRUE_POP_r32 }, + }, + }, + [_GUARD_IS_FALSE_POP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IS_FALSE_POP_r00 }, + { 0, 0, _GUARD_IS_FALSE_POP_r10 }, + { 1, 1, _GUARD_IS_FALSE_POP_r21 }, + { 2, 2, _GUARD_IS_FALSE_POP_r32 }, + }, + }, + [_GUARD_BIT_IS_SET_POP_4] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_SET_POP_4_r00 }, + { 0, 0, _GUARD_BIT_IS_SET_POP_4_r10 }, + { 1, 1, _GUARD_BIT_IS_SET_POP_4_r21 }, + { 2, 2, _GUARD_BIT_IS_SET_POP_4_r32 }, + }, + }, + [_GUARD_BIT_IS_SET_POP_5] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_SET_POP_5_r00 }, + { 0, 0, _GUARD_BIT_IS_SET_POP_5_r10 }, + { 1, 1, _GUARD_BIT_IS_SET_POP_5_r21 }, + { 2, 2, _GUARD_BIT_IS_SET_POP_5_r32 }, + }, + }, + [_GUARD_BIT_IS_SET_POP_6] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_SET_POP_6_r00 }, + { 0, 0, _GUARD_BIT_IS_SET_POP_6_r10 }, + { 1, 1, _GUARD_BIT_IS_SET_POP_6_r21 }, + { 2, 2, _GUARD_BIT_IS_SET_POP_6_r32 }, + }, + }, + [_GUARD_BIT_IS_SET_POP_7] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_SET_POP_7_r00 }, + { 0, 0, _GUARD_BIT_IS_SET_POP_7_r10 }, + { 1, 1, _GUARD_BIT_IS_SET_POP_7_r21 }, + { 2, 2, _GUARD_BIT_IS_SET_POP_7_r32 }, + }, + }, + [_GUARD_BIT_IS_SET_POP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_SET_POP_r00 }, + { 0, 0, _GUARD_BIT_IS_SET_POP_r10 }, + { 1, 1, _GUARD_BIT_IS_SET_POP_r21 }, + { 2, 2, _GUARD_BIT_IS_SET_POP_r32 }, + }, + }, + [_GUARD_BIT_IS_UNSET_POP_4] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_UNSET_POP_4_r00 }, + { 0, 0, _GUARD_BIT_IS_UNSET_POP_4_r10 }, + { 1, 1, _GUARD_BIT_IS_UNSET_POP_4_r21 }, + { 2, 2, _GUARD_BIT_IS_UNSET_POP_4_r32 }, + }, + }, + [_GUARD_BIT_IS_UNSET_POP_5] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_UNSET_POP_5_r00 }, + { 0, 0, _GUARD_BIT_IS_UNSET_POP_5_r10 }, + { 1, 1, _GUARD_BIT_IS_UNSET_POP_5_r21 }, + { 2, 2, _GUARD_BIT_IS_UNSET_POP_5_r32 }, + }, + }, + [_GUARD_BIT_IS_UNSET_POP_6] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_UNSET_POP_6_r00 }, + { 0, 0, _GUARD_BIT_IS_UNSET_POP_6_r10 }, + { 1, 1, _GUARD_BIT_IS_UNSET_POP_6_r21 }, + { 2, 2, _GUARD_BIT_IS_UNSET_POP_6_r32 }, + }, + }, + [_GUARD_BIT_IS_UNSET_POP_7] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_UNSET_POP_7_r00 }, + { 0, 0, _GUARD_BIT_IS_UNSET_POP_7_r10 }, + { 1, 1, _GUARD_BIT_IS_UNSET_POP_7_r21 }, + { 2, 2, _GUARD_BIT_IS_UNSET_POP_7_r32 }, + }, + }, + [_GUARD_BIT_IS_UNSET_POP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_UNSET_POP_r00 }, + { 0, 0, _GUARD_BIT_IS_UNSET_POP_r10 }, + { 1, 1, _GUARD_BIT_IS_UNSET_POP_r21 }, + { 2, 2, _GUARD_BIT_IS_UNSET_POP_r32 }, + }, + }, + [_GUARD_IS_NONE_POP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IS_NONE_POP_r00 }, + { 0, 0, _GUARD_IS_NONE_POP_r10 }, + { 1, 1, _GUARD_IS_NONE_POP_r21 }, + { 2, 2, _GUARD_IS_NONE_POP_r32 }, + }, + }, + [_GUARD_IS_NOT_NONE_POP] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _GUARD_IS_NOT_NONE_POP_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_JUMP_TO_TOP] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _JUMP_TO_TOP_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SET_IP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _SET_IP_r00 }, + { 1, 1, _SET_IP_r11 }, + { 2, 2, _SET_IP_r22 }, + { 3, 3, _SET_IP_r33 }, + }, + }, + [_CHECK_STACK_SPACE_OPERAND] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_STACK_SPACE_OPERAND_r00 }, + { 1, 1, _CHECK_STACK_SPACE_OPERAND_r11 }, + { 2, 2, _CHECK_STACK_SPACE_OPERAND_r22 }, + { 3, 3, _CHECK_STACK_SPACE_OPERAND_r33 }, + }, + }, + [_SAVE_RETURN_OFFSET] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _SAVE_RETURN_OFFSET_r00 }, + { 1, 1, _SAVE_RETURN_OFFSET_r11 }, + { 2, 2, _SAVE_RETURN_OFFSET_r22 }, + { 3, 3, _SAVE_RETURN_OFFSET_r33 }, + }, + }, + [_EXIT_TRACE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _EXIT_TRACE_r00 }, + { 0, 0, _EXIT_TRACE_r10 }, + { 0, 0, _EXIT_TRACE_r20 }, + { 0, 0, _EXIT_TRACE_r30 }, + }, + }, + [_DYNAMIC_EXIT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _DYNAMIC_EXIT_r00 }, + { 0, 0, _DYNAMIC_EXIT_r10 }, + { 0, 0, _DYNAMIC_EXIT_r20 }, + { 0, 0, _DYNAMIC_EXIT_r30 }, + }, + }, + [_CHECK_VALIDITY] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_VALIDITY_r00 }, + { 1, 1, _CHECK_VALIDITY_r11 }, + { 2, 2, _CHECK_VALIDITY_r22 }, + { 3, 3, _CHECK_VALIDITY_r33 }, + }, + }, + [_LOAD_CONST_INLINE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_CONST_INLINE_r01 }, + { 2, 1, _LOAD_CONST_INLINE_r12 }, + { 3, 2, _LOAD_CONST_INLINE_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_CONST_INLINE_BORROW] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_CONST_INLINE_BORROW_r01 }, + { 2, 1, _LOAD_CONST_INLINE_BORROW_r12 }, + { 3, 2, _LOAD_CONST_INLINE_BORROW_r23 }, + { -1, -1, -1 }, + }, + }, + [_RROT_3] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _RROT_3_r03 }, + { 3, 1, _RROT_3_r13 }, + { 3, 2, _RROT_3_r23 }, + { 3, 3, _RROT_3_r33 }, + }, + }, + [_START_EXECUTOR] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _START_EXECUTOR_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAKE_WARM] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _MAKE_WARM_r00 }, + { 1, 1, _MAKE_WARM_r11 }, + { 2, 2, _MAKE_WARM_r22 }, + { 3, 3, _MAKE_WARM_r33 }, + }, + }, + [_FATAL_ERROR] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _FATAL_ERROR_r00 }, + { 1, 1, _FATAL_ERROR_r11 }, + { 2, 2, _FATAL_ERROR_r22 }, + { 3, 3, _FATAL_ERROR_r33 }, + }, + }, + [_DEOPT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _DEOPT_r00 }, + { 0, 0, _DEOPT_r10 }, + { 0, 0, _DEOPT_r20 }, + { 0, 0, _DEOPT_r30 }, + }, + }, + [_HANDLE_PENDING_AND_DEOPT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _HANDLE_PENDING_AND_DEOPT_r00 }, + { 0, 0, _HANDLE_PENDING_AND_DEOPT_r10 }, + { 0, 0, _HANDLE_PENDING_AND_DEOPT_r20 }, + { 0, 0, _HANDLE_PENDING_AND_DEOPT_r30 }, + }, + }, + [_ERROR_POP_N] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _ERROR_POP_N_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_TIER2_RESUME_CHECK] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _TIER2_RESUME_CHECK_r00 }, + { 1, 1, _TIER2_RESUME_CHECK_r11 }, + { 2, 2, _TIER2_RESUME_CHECK_r22 }, + { 3, 3, _TIER2_RESUME_CHECK_r33 }, + }, + }, + [_COLD_EXIT] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _COLD_EXIT_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_COLD_DYNAMIC_EXIT] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _COLD_DYNAMIC_EXIT_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CODE_VERSION__PUSH_FRAME] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_CODE_VERSION__PUSH_FRAME_r00 }, + { 1, 1, _GUARD_CODE_VERSION__PUSH_FRAME_r11 }, + { 2, 2, _GUARD_CODE_VERSION__PUSH_FRAME_r22 }, + { 3, 3, _GUARD_CODE_VERSION__PUSH_FRAME_r33 }, + }, + }, + [_GUARD_CODE_VERSION_YIELD_VALUE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_CODE_VERSION_YIELD_VALUE_r00 }, + { 1, 1, _GUARD_CODE_VERSION_YIELD_VALUE_r11 }, + { 2, 2, _GUARD_CODE_VERSION_YIELD_VALUE_r22 }, + { 3, 3, _GUARD_CODE_VERSION_YIELD_VALUE_r33 }, + }, + }, + [_GUARD_CODE_VERSION_RETURN_VALUE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_CODE_VERSION_RETURN_VALUE_r00 }, + { 1, 1, _GUARD_CODE_VERSION_RETURN_VALUE_r11 }, + { 2, 2, _GUARD_CODE_VERSION_RETURN_VALUE_r22 }, + { 3, 3, _GUARD_CODE_VERSION_RETURN_VALUE_r33 }, + }, + }, + [_GUARD_CODE_VERSION_RETURN_GENERATOR] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_CODE_VERSION_RETURN_GENERATOR_r00 }, + { 1, 1, _GUARD_CODE_VERSION_RETURN_GENERATOR_r11 }, + { 2, 2, _GUARD_CODE_VERSION_RETURN_GENERATOR_r22 }, + { 3, 3, _GUARD_CODE_VERSION_RETURN_GENERATOR_r33 }, + }, + }, + [_GUARD_IP__PUSH_FRAME] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IP__PUSH_FRAME_r00 }, + { 1, 1, _GUARD_IP__PUSH_FRAME_r11 }, + { 2, 2, _GUARD_IP__PUSH_FRAME_r22 }, + { 3, 3, _GUARD_IP__PUSH_FRAME_r33 }, + }, + }, + [_GUARD_IP_YIELD_VALUE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IP_YIELD_VALUE_r00 }, + { 1, 1, _GUARD_IP_YIELD_VALUE_r11 }, + { 2, 2, _GUARD_IP_YIELD_VALUE_r22 }, + { 3, 3, _GUARD_IP_YIELD_VALUE_r33 }, + }, + }, + [_GUARD_IP_RETURN_VALUE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IP_RETURN_VALUE_r00 }, + { 1, 1, _GUARD_IP_RETURN_VALUE_r11 }, + { 2, 2, _GUARD_IP_RETURN_VALUE_r22 }, + { 3, 3, _GUARD_IP_RETURN_VALUE_r33 }, + }, + }, + [_GUARD_IP_RETURN_GENERATOR] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IP_RETURN_GENERATOR_r00 }, + { 1, 1, _GUARD_IP_RETURN_GENERATOR_r11 }, + { 2, 2, _GUARD_IP_RETURN_GENERATOR_r22 }, + { 3, 3, _GUARD_IP_RETURN_GENERATOR_r33 }, + }, + }, +}; + +const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { + [_NOP_r00] = _NOP, + [_NOP_r11] = _NOP, + [_NOP_r22] = _NOP, + [_NOP_r33] = _NOP, + [_CHECK_PERIODIC_r00] = _CHECK_PERIODIC, + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00] = _CHECK_PERIODIC_IF_NOT_YIELD_FROM, + [_RESUME_CHECK_r00] = _RESUME_CHECK, + [_RESUME_CHECK_r11] = _RESUME_CHECK, + [_RESUME_CHECK_r22] = _RESUME_CHECK, + [_RESUME_CHECK_r33] = _RESUME_CHECK, + [_LOAD_FAST_CHECK_r01] = _LOAD_FAST_CHECK, + [_LOAD_FAST_CHECK_r12] = _LOAD_FAST_CHECK, + [_LOAD_FAST_CHECK_r23] = _LOAD_FAST_CHECK, + [_LOAD_FAST_0_r01] = _LOAD_FAST_0, + [_LOAD_FAST_0_r12] = _LOAD_FAST_0, + [_LOAD_FAST_0_r23] = _LOAD_FAST_0, + [_LOAD_FAST_1_r01] = _LOAD_FAST_1, + [_LOAD_FAST_1_r12] = _LOAD_FAST_1, + [_LOAD_FAST_1_r23] = _LOAD_FAST_1, + [_LOAD_FAST_2_r01] = _LOAD_FAST_2, + [_LOAD_FAST_2_r12] = _LOAD_FAST_2, + [_LOAD_FAST_2_r23] = _LOAD_FAST_2, + [_LOAD_FAST_3_r01] = _LOAD_FAST_3, + [_LOAD_FAST_3_r12] = _LOAD_FAST_3, + [_LOAD_FAST_3_r23] = _LOAD_FAST_3, + [_LOAD_FAST_4_r01] = _LOAD_FAST_4, + [_LOAD_FAST_4_r12] = _LOAD_FAST_4, + [_LOAD_FAST_4_r23] = _LOAD_FAST_4, + [_LOAD_FAST_5_r01] = _LOAD_FAST_5, + [_LOAD_FAST_5_r12] = _LOAD_FAST_5, + [_LOAD_FAST_5_r23] = _LOAD_FAST_5, + [_LOAD_FAST_6_r01] = _LOAD_FAST_6, + [_LOAD_FAST_6_r12] = _LOAD_FAST_6, + [_LOAD_FAST_6_r23] = _LOAD_FAST_6, + [_LOAD_FAST_7_r01] = _LOAD_FAST_7, + [_LOAD_FAST_7_r12] = _LOAD_FAST_7, + [_LOAD_FAST_7_r23] = _LOAD_FAST_7, + [_LOAD_FAST_r01] = _LOAD_FAST, + [_LOAD_FAST_r12] = _LOAD_FAST, + [_LOAD_FAST_r23] = _LOAD_FAST, + [_LOAD_FAST_BORROW_0_r01] = _LOAD_FAST_BORROW_0, + [_LOAD_FAST_BORROW_0_r12] = _LOAD_FAST_BORROW_0, + [_LOAD_FAST_BORROW_0_r23] = _LOAD_FAST_BORROW_0, + [_LOAD_FAST_BORROW_1_r01] = _LOAD_FAST_BORROW_1, + [_LOAD_FAST_BORROW_1_r12] = _LOAD_FAST_BORROW_1, + [_LOAD_FAST_BORROW_1_r23] = _LOAD_FAST_BORROW_1, + [_LOAD_FAST_BORROW_2_r01] = _LOAD_FAST_BORROW_2, + [_LOAD_FAST_BORROW_2_r12] = _LOAD_FAST_BORROW_2, + [_LOAD_FAST_BORROW_2_r23] = _LOAD_FAST_BORROW_2, + [_LOAD_FAST_BORROW_3_r01] = _LOAD_FAST_BORROW_3, + [_LOAD_FAST_BORROW_3_r12] = _LOAD_FAST_BORROW_3, + [_LOAD_FAST_BORROW_3_r23] = _LOAD_FAST_BORROW_3, + [_LOAD_FAST_BORROW_4_r01] = _LOAD_FAST_BORROW_4, + [_LOAD_FAST_BORROW_4_r12] = _LOAD_FAST_BORROW_4, + [_LOAD_FAST_BORROW_4_r23] = _LOAD_FAST_BORROW_4, + [_LOAD_FAST_BORROW_5_r01] = _LOAD_FAST_BORROW_5, + [_LOAD_FAST_BORROW_5_r12] = _LOAD_FAST_BORROW_5, + [_LOAD_FAST_BORROW_5_r23] = _LOAD_FAST_BORROW_5, + [_LOAD_FAST_BORROW_6_r01] = _LOAD_FAST_BORROW_6, + [_LOAD_FAST_BORROW_6_r12] = _LOAD_FAST_BORROW_6, + [_LOAD_FAST_BORROW_6_r23] = _LOAD_FAST_BORROW_6, + [_LOAD_FAST_BORROW_7_r01] = _LOAD_FAST_BORROW_7, + [_LOAD_FAST_BORROW_7_r12] = _LOAD_FAST_BORROW_7, + [_LOAD_FAST_BORROW_7_r23] = _LOAD_FAST_BORROW_7, + [_LOAD_FAST_BORROW_r01] = _LOAD_FAST_BORROW, + [_LOAD_FAST_BORROW_r12] = _LOAD_FAST_BORROW, + [_LOAD_FAST_BORROW_r23] = _LOAD_FAST_BORROW, + [_LOAD_FAST_AND_CLEAR_r01] = _LOAD_FAST_AND_CLEAR, + [_LOAD_FAST_AND_CLEAR_r12] = _LOAD_FAST_AND_CLEAR, + [_LOAD_FAST_AND_CLEAR_r23] = _LOAD_FAST_AND_CLEAR, + [_LOAD_CONST_r01] = _LOAD_CONST, + [_LOAD_CONST_r12] = _LOAD_CONST, + [_LOAD_CONST_r23] = _LOAD_CONST, + [_LOAD_SMALL_INT_0_r01] = _LOAD_SMALL_INT_0, + [_LOAD_SMALL_INT_0_r12] = _LOAD_SMALL_INT_0, + [_LOAD_SMALL_INT_0_r23] = _LOAD_SMALL_INT_0, + [_LOAD_SMALL_INT_1_r01] = _LOAD_SMALL_INT_1, + [_LOAD_SMALL_INT_1_r12] = _LOAD_SMALL_INT_1, + [_LOAD_SMALL_INT_1_r23] = _LOAD_SMALL_INT_1, + [_LOAD_SMALL_INT_2_r01] = _LOAD_SMALL_INT_2, + [_LOAD_SMALL_INT_2_r12] = _LOAD_SMALL_INT_2, + [_LOAD_SMALL_INT_2_r23] = _LOAD_SMALL_INT_2, + [_LOAD_SMALL_INT_3_r01] = _LOAD_SMALL_INT_3, + [_LOAD_SMALL_INT_3_r12] = _LOAD_SMALL_INT_3, + [_LOAD_SMALL_INT_3_r23] = _LOAD_SMALL_INT_3, + [_LOAD_SMALL_INT_r01] = _LOAD_SMALL_INT, + [_LOAD_SMALL_INT_r12] = _LOAD_SMALL_INT, + [_LOAD_SMALL_INT_r23] = _LOAD_SMALL_INT, + [_SWAP_FAST_0_r01] = _SWAP_FAST_0, + [_SWAP_FAST_0_r11] = _SWAP_FAST_0, + [_SWAP_FAST_0_r22] = _SWAP_FAST_0, + [_SWAP_FAST_0_r33] = _SWAP_FAST_0, + [_SWAP_FAST_1_r01] = _SWAP_FAST_1, + [_SWAP_FAST_1_r11] = _SWAP_FAST_1, + [_SWAP_FAST_1_r22] = _SWAP_FAST_1, + [_SWAP_FAST_1_r33] = _SWAP_FAST_1, + [_SWAP_FAST_2_r01] = _SWAP_FAST_2, + [_SWAP_FAST_2_r11] = _SWAP_FAST_2, + [_SWAP_FAST_2_r22] = _SWAP_FAST_2, + [_SWAP_FAST_2_r33] = _SWAP_FAST_2, + [_SWAP_FAST_3_r01] = _SWAP_FAST_3, + [_SWAP_FAST_3_r11] = _SWAP_FAST_3, + [_SWAP_FAST_3_r22] = _SWAP_FAST_3, + [_SWAP_FAST_3_r33] = _SWAP_FAST_3, + [_SWAP_FAST_4_r01] = _SWAP_FAST_4, + [_SWAP_FAST_4_r11] = _SWAP_FAST_4, + [_SWAP_FAST_4_r22] = _SWAP_FAST_4, + [_SWAP_FAST_4_r33] = _SWAP_FAST_4, + [_SWAP_FAST_5_r01] = _SWAP_FAST_5, + [_SWAP_FAST_5_r11] = _SWAP_FAST_5, + [_SWAP_FAST_5_r22] = _SWAP_FAST_5, + [_SWAP_FAST_5_r33] = _SWAP_FAST_5, + [_SWAP_FAST_6_r01] = _SWAP_FAST_6, + [_SWAP_FAST_6_r11] = _SWAP_FAST_6, + [_SWAP_FAST_6_r22] = _SWAP_FAST_6, + [_SWAP_FAST_6_r33] = _SWAP_FAST_6, + [_SWAP_FAST_7_r01] = _SWAP_FAST_7, + [_SWAP_FAST_7_r11] = _SWAP_FAST_7, + [_SWAP_FAST_7_r22] = _SWAP_FAST_7, + [_SWAP_FAST_7_r33] = _SWAP_FAST_7, + [_SWAP_FAST_r01] = _SWAP_FAST, + [_SWAP_FAST_r11] = _SWAP_FAST, + [_SWAP_FAST_r22] = _SWAP_FAST, + [_SWAP_FAST_r33] = _SWAP_FAST, + [_POP_TOP_r10] = _POP_TOP, + [_POP_TOP_NOP_r00] = _POP_TOP_NOP, + [_POP_TOP_NOP_r10] = _POP_TOP_NOP, + [_POP_TOP_NOP_r21] = _POP_TOP_NOP, + [_POP_TOP_NOP_r32] = _POP_TOP_NOP, + [_POP_TOP_INT_r00] = _POP_TOP_INT, + [_POP_TOP_INT_r10] = _POP_TOP_INT, + [_POP_TOP_INT_r21] = _POP_TOP_INT, + [_POP_TOP_INT_r32] = _POP_TOP_INT, + [_POP_TOP_FLOAT_r00] = _POP_TOP_FLOAT, + [_POP_TOP_FLOAT_r10] = _POP_TOP_FLOAT, + [_POP_TOP_FLOAT_r21] = _POP_TOP_FLOAT, + [_POP_TOP_FLOAT_r32] = _POP_TOP_FLOAT, + [_POP_TOP_UNICODE_r00] = _POP_TOP_UNICODE, + [_POP_TOP_UNICODE_r10] = _POP_TOP_UNICODE, + [_POP_TOP_UNICODE_r21] = _POP_TOP_UNICODE, + [_POP_TOP_UNICODE_r32] = _POP_TOP_UNICODE, + [_POP_TOP_OPARG_r00] = _POP_TOP_OPARG, + [_PUSH_NULL_r01] = _PUSH_NULL, + [_PUSH_NULL_r12] = _PUSH_NULL, + [_PUSH_NULL_r23] = _PUSH_NULL, + [_END_FOR_r10] = _END_FOR, + [_POP_ITER_r20] = _POP_ITER, + [_END_SEND_r31] = _END_SEND, + [_UNARY_NEGATIVE_r12] = _UNARY_NEGATIVE, + [_UNARY_NEGATIVE_FLOAT_INPLACE_r02] = _UNARY_NEGATIVE_FLOAT_INPLACE, + [_UNARY_NEGATIVE_FLOAT_INPLACE_r12] = _UNARY_NEGATIVE_FLOAT_INPLACE, + [_UNARY_NEGATIVE_FLOAT_INPLACE_r23] = _UNARY_NEGATIVE_FLOAT_INPLACE, + [_UNARY_NOT_r01] = _UNARY_NOT, + [_UNARY_NOT_r11] = _UNARY_NOT, + [_UNARY_NOT_r22] = _UNARY_NOT, + [_UNARY_NOT_r33] = _UNARY_NOT, + [_TO_BOOL_r11] = _TO_BOOL, + [_TO_BOOL_BOOL_r01] = _TO_BOOL_BOOL, + [_TO_BOOL_BOOL_r11] = _TO_BOOL_BOOL, + [_TO_BOOL_BOOL_r22] = _TO_BOOL_BOOL, + [_TO_BOOL_BOOL_r33] = _TO_BOOL_BOOL, + [_TO_BOOL_INT_r02] = _TO_BOOL_INT, + [_TO_BOOL_INT_r12] = _TO_BOOL_INT, + [_TO_BOOL_INT_r23] = _TO_BOOL_INT, + [_GUARD_NOS_LIST_r02] = _GUARD_NOS_LIST, + [_GUARD_NOS_LIST_r12] = _GUARD_NOS_LIST, + [_GUARD_NOS_LIST_r22] = _GUARD_NOS_LIST, + [_GUARD_NOS_LIST_r33] = _GUARD_NOS_LIST, + [_GUARD_TOS_LIST_r01] = _GUARD_TOS_LIST, + [_GUARD_TOS_LIST_r11] = _GUARD_TOS_LIST, + [_GUARD_TOS_LIST_r22] = _GUARD_TOS_LIST, + [_GUARD_TOS_LIST_r33] = _GUARD_TOS_LIST, + [_GUARD_TOS_SLICE_r01] = _GUARD_TOS_SLICE, + [_GUARD_TOS_SLICE_r11] = _GUARD_TOS_SLICE, + [_GUARD_TOS_SLICE_r22] = _GUARD_TOS_SLICE, + [_GUARD_TOS_SLICE_r33] = _GUARD_TOS_SLICE, + [_TO_BOOL_LIST_r02] = _TO_BOOL_LIST, + [_TO_BOOL_LIST_r12] = _TO_BOOL_LIST, + [_TO_BOOL_LIST_r23] = _TO_BOOL_LIST, + [_TO_BOOL_NONE_r01] = _TO_BOOL_NONE, + [_TO_BOOL_NONE_r11] = _TO_BOOL_NONE, + [_TO_BOOL_NONE_r22] = _TO_BOOL_NONE, + [_TO_BOOL_NONE_r33] = _TO_BOOL_NONE, + [_GUARD_NOS_COMPACT_ASCII_r02] = _GUARD_NOS_COMPACT_ASCII, + [_GUARD_NOS_COMPACT_ASCII_r12] = _GUARD_NOS_COMPACT_ASCII, + [_GUARD_NOS_COMPACT_ASCII_r22] = _GUARD_NOS_COMPACT_ASCII, + [_GUARD_NOS_COMPACT_ASCII_r33] = _GUARD_NOS_COMPACT_ASCII, + [_GUARD_NOS_UNICODE_r02] = _GUARD_NOS_UNICODE, + [_GUARD_NOS_UNICODE_r12] = _GUARD_NOS_UNICODE, + [_GUARD_NOS_UNICODE_r22] = _GUARD_NOS_UNICODE, + [_GUARD_NOS_UNICODE_r33] = _GUARD_NOS_UNICODE, + [_GUARD_TOS_UNICODE_r01] = _GUARD_TOS_UNICODE, + [_GUARD_TOS_UNICODE_r11] = _GUARD_TOS_UNICODE, + [_GUARD_TOS_UNICODE_r22] = _GUARD_TOS_UNICODE, + [_GUARD_TOS_UNICODE_r33] = _GUARD_TOS_UNICODE, + [_TO_BOOL_STR_r02] = _TO_BOOL_STR, + [_TO_BOOL_STR_r12] = _TO_BOOL_STR, + [_TO_BOOL_STR_r23] = _TO_BOOL_STR, + [_REPLACE_WITH_TRUE_r02] = _REPLACE_WITH_TRUE, + [_REPLACE_WITH_TRUE_r12] = _REPLACE_WITH_TRUE, + [_REPLACE_WITH_TRUE_r23] = _REPLACE_WITH_TRUE, + [_UNARY_INVERT_r12] = _UNARY_INVERT, + [_GUARD_NOS_INT_r02] = _GUARD_NOS_INT, + [_GUARD_NOS_INT_r12] = _GUARD_NOS_INT, + [_GUARD_NOS_INT_r22] = _GUARD_NOS_INT, + [_GUARD_NOS_INT_r33] = _GUARD_NOS_INT, + [_GUARD_TOS_INT_r01] = _GUARD_TOS_INT, + [_GUARD_TOS_INT_r11] = _GUARD_TOS_INT, + [_GUARD_TOS_INT_r22] = _GUARD_TOS_INT, + [_GUARD_TOS_INT_r33] = _GUARD_TOS_INT, + [_GUARD_NOS_OVERFLOWED_r02] = _GUARD_NOS_OVERFLOWED, + [_GUARD_NOS_OVERFLOWED_r12] = _GUARD_NOS_OVERFLOWED, + [_GUARD_NOS_OVERFLOWED_r22] = _GUARD_NOS_OVERFLOWED, + [_GUARD_NOS_OVERFLOWED_r33] = _GUARD_NOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r01] = _GUARD_TOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r11] = _GUARD_TOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r22] = _GUARD_TOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r33] = _GUARD_TOS_OVERFLOWED, + [_BINARY_OP_MULTIPLY_INT_r03] = _BINARY_OP_MULTIPLY_INT, + [_BINARY_OP_MULTIPLY_INT_r13] = _BINARY_OP_MULTIPLY_INT, + [_BINARY_OP_MULTIPLY_INT_r23] = _BINARY_OP_MULTIPLY_INT, + [_BINARY_OP_ADD_INT_r03] = _BINARY_OP_ADD_INT, + [_BINARY_OP_ADD_INT_r13] = _BINARY_OP_ADD_INT, + [_BINARY_OP_ADD_INT_r23] = _BINARY_OP_ADD_INT, + [_BINARY_OP_SUBTRACT_INT_r03] = _BINARY_OP_SUBTRACT_INT, + [_BINARY_OP_SUBTRACT_INT_r13] = _BINARY_OP_SUBTRACT_INT, + [_BINARY_OP_SUBTRACT_INT_r23] = _BINARY_OP_SUBTRACT_INT, + [_BINARY_OP_ADD_INT_INPLACE_r03] = _BINARY_OP_ADD_INT_INPLACE, + [_BINARY_OP_ADD_INT_INPLACE_r13] = _BINARY_OP_ADD_INT_INPLACE, + [_BINARY_OP_ADD_INT_INPLACE_r23] = _BINARY_OP_ADD_INT_INPLACE, + [_BINARY_OP_SUBTRACT_INT_INPLACE_r03] = _BINARY_OP_SUBTRACT_INT_INPLACE, + [_BINARY_OP_SUBTRACT_INT_INPLACE_r13] = _BINARY_OP_SUBTRACT_INT_INPLACE, + [_BINARY_OP_SUBTRACT_INT_INPLACE_r23] = _BINARY_OP_SUBTRACT_INT_INPLACE, + [_BINARY_OP_MULTIPLY_INT_INPLACE_r03] = _BINARY_OP_MULTIPLY_INT_INPLACE, + [_BINARY_OP_MULTIPLY_INT_INPLACE_r13] = _BINARY_OP_MULTIPLY_INT_INPLACE, + [_BINARY_OP_MULTIPLY_INT_INPLACE_r23] = _BINARY_OP_MULTIPLY_INT_INPLACE, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r03] = _BINARY_OP_ADD_INT_INPLACE_RIGHT, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r13] = _BINARY_OP_ADD_INT_INPLACE_RIGHT, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r23] = _BINARY_OP_ADD_INT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03] = _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13] = _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23] = _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03] = _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13] = _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23] = _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, + [_GUARD_NOS_FLOAT_r02] = _GUARD_NOS_FLOAT, + [_GUARD_NOS_FLOAT_r12] = _GUARD_NOS_FLOAT, + [_GUARD_NOS_FLOAT_r22] = _GUARD_NOS_FLOAT, + [_GUARD_NOS_FLOAT_r33] = _GUARD_NOS_FLOAT, + [_GUARD_TOS_FLOAT_r01] = _GUARD_TOS_FLOAT, + [_GUARD_TOS_FLOAT_r11] = _GUARD_TOS_FLOAT, + [_GUARD_TOS_FLOAT_r22] = _GUARD_TOS_FLOAT, + [_GUARD_TOS_FLOAT_r33] = _GUARD_TOS_FLOAT, + [_BINARY_OP_MULTIPLY_FLOAT_r03] = _BINARY_OP_MULTIPLY_FLOAT, + [_BINARY_OP_MULTIPLY_FLOAT_r13] = _BINARY_OP_MULTIPLY_FLOAT, + [_BINARY_OP_MULTIPLY_FLOAT_r23] = _BINARY_OP_MULTIPLY_FLOAT, + [_BINARY_OP_ADD_FLOAT_r03] = _BINARY_OP_ADD_FLOAT, + [_BINARY_OP_ADD_FLOAT_r13] = _BINARY_OP_ADD_FLOAT, + [_BINARY_OP_ADD_FLOAT_r23] = _BINARY_OP_ADD_FLOAT, + [_BINARY_OP_SUBTRACT_FLOAT_r03] = _BINARY_OP_SUBTRACT_FLOAT, + [_BINARY_OP_SUBTRACT_FLOAT_r13] = _BINARY_OP_SUBTRACT_FLOAT, + [_BINARY_OP_SUBTRACT_FLOAT_r23] = _BINARY_OP_SUBTRACT_FLOAT, + [_BINARY_OP_ADD_FLOAT_INPLACE_r03] = _BINARY_OP_ADD_FLOAT_INPLACE, + [_BINARY_OP_ADD_FLOAT_INPLACE_r13] = _BINARY_OP_ADD_FLOAT_INPLACE, + [_BINARY_OP_ADD_FLOAT_INPLACE_r23] = _BINARY_OP_ADD_FLOAT_INPLACE, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03] = _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13] = _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23] = _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_TRUEDIV_FLOAT_r23] = _BINARY_OP_TRUEDIV_FLOAT, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_ADD_UNICODE_r03] = _BINARY_OP_ADD_UNICODE, + [_BINARY_OP_ADD_UNICODE_r13] = _BINARY_OP_ADD_UNICODE, + [_BINARY_OP_ADD_UNICODE_r23] = _BINARY_OP_ADD_UNICODE, + [_BINARY_OP_INPLACE_ADD_UNICODE_r21] = _BINARY_OP_INPLACE_ADD_UNICODE, + [_GUARD_BINARY_OP_EXTEND_LHS_r02] = _GUARD_BINARY_OP_EXTEND_LHS, + [_GUARD_BINARY_OP_EXTEND_LHS_r12] = _GUARD_BINARY_OP_EXTEND_LHS, + [_GUARD_BINARY_OP_EXTEND_LHS_r22] = _GUARD_BINARY_OP_EXTEND_LHS, + [_GUARD_BINARY_OP_EXTEND_LHS_r33] = _GUARD_BINARY_OP_EXTEND_LHS, + [_GUARD_BINARY_OP_EXTEND_RHS_r02] = _GUARD_BINARY_OP_EXTEND_RHS, + [_GUARD_BINARY_OP_EXTEND_RHS_r12] = _GUARD_BINARY_OP_EXTEND_RHS, + [_GUARD_BINARY_OP_EXTEND_RHS_r22] = _GUARD_BINARY_OP_EXTEND_RHS, + [_GUARD_BINARY_OP_EXTEND_RHS_r33] = _GUARD_BINARY_OP_EXTEND_RHS, + [_GUARD_BINARY_OP_EXTEND_r22] = _GUARD_BINARY_OP_EXTEND, + [_BINARY_OP_EXTEND_r23] = _BINARY_OP_EXTEND, + [_BINARY_SLICE_r31] = _BINARY_SLICE, + [_STORE_SLICE_r30] = _STORE_SLICE, + [_BINARY_OP_SUBSCR_LIST_INT_r23] = _BINARY_OP_SUBSCR_LIST_INT, + [_BINARY_OP_SUBSCR_LIST_SLICE_r23] = _BINARY_OP_SUBSCR_LIST_SLICE, + [_BINARY_OP_SUBSCR_STR_INT_r23] = _BINARY_OP_SUBSCR_STR_INT, + [_BINARY_OP_SUBSCR_USTR_INT_r23] = _BINARY_OP_SUBSCR_USTR_INT, + [_GUARD_NOS_TUPLE_r02] = _GUARD_NOS_TUPLE, + [_GUARD_NOS_TUPLE_r12] = _GUARD_NOS_TUPLE, + [_GUARD_NOS_TUPLE_r22] = _GUARD_NOS_TUPLE, + [_GUARD_NOS_TUPLE_r33] = _GUARD_NOS_TUPLE, + [_GUARD_TOS_TUPLE_r01] = _GUARD_TOS_TUPLE, + [_GUARD_TOS_TUPLE_r11] = _GUARD_TOS_TUPLE, + [_GUARD_TOS_TUPLE_r22] = _GUARD_TOS_TUPLE, + [_GUARD_TOS_TUPLE_r33] = _GUARD_TOS_TUPLE, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02] = _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12] = _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22] = _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33] = _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, + [_BINARY_OP_SUBSCR_TUPLE_INT_r03] = _BINARY_OP_SUBSCR_TUPLE_INT, + [_BINARY_OP_SUBSCR_TUPLE_INT_r13] = _BINARY_OP_SUBSCR_TUPLE_INT, + [_BINARY_OP_SUBSCR_TUPLE_INT_r23] = _BINARY_OP_SUBSCR_TUPLE_INT, + [_GUARD_NOS_DICT_SUBSCRIPT_r02] = _GUARD_NOS_DICT_SUBSCRIPT, + [_GUARD_NOS_DICT_SUBSCRIPT_r12] = _GUARD_NOS_DICT_SUBSCRIPT, + [_GUARD_NOS_DICT_SUBSCRIPT_r22] = _GUARD_NOS_DICT_SUBSCRIPT, + [_GUARD_NOS_DICT_SUBSCRIPT_r33] = _GUARD_NOS_DICT_SUBSCRIPT, + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r03] = _GUARD_NOS_DICT_STORE_SUBSCRIPT, + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r13] = _GUARD_NOS_DICT_STORE_SUBSCRIPT, + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r23] = _GUARD_NOS_DICT_STORE_SUBSCRIPT, + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r33] = _GUARD_NOS_DICT_STORE_SUBSCRIPT, + [_GUARD_TOS_ANY_DICT_r01] = _GUARD_TOS_ANY_DICT, + [_GUARD_TOS_ANY_DICT_r11] = _GUARD_TOS_ANY_DICT, + [_GUARD_TOS_ANY_DICT_r22] = _GUARD_TOS_ANY_DICT, + [_GUARD_TOS_ANY_DICT_r33] = _GUARD_TOS_ANY_DICT, + [_GUARD_TOS_DICT_r01] = _GUARD_TOS_DICT, + [_GUARD_TOS_DICT_r11] = _GUARD_TOS_DICT, + [_GUARD_TOS_DICT_r22] = _GUARD_TOS_DICT, + [_GUARD_TOS_DICT_r33] = _GUARD_TOS_DICT, + [_GUARD_TOS_FROZENDICT_r01] = _GUARD_TOS_FROZENDICT, + [_GUARD_TOS_FROZENDICT_r11] = _GUARD_TOS_FROZENDICT, + [_GUARD_TOS_FROZENDICT_r22] = _GUARD_TOS_FROZENDICT, + [_GUARD_TOS_FROZENDICT_r33] = _GUARD_TOS_FROZENDICT, + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23] = _BINARY_OP_SUBSCR_DICT_KNOWN_HASH, + [_BINARY_OP_SUBSCR_DICT_r23] = _BINARY_OP_SUBSCR_DICT, + [_BINARY_OP_SUBSCR_CHECK_FUNC_r23] = _BINARY_OP_SUBSCR_CHECK_FUNC, + [_BINARY_OP_SUBSCR_INIT_CALL_r01] = _BINARY_OP_SUBSCR_INIT_CALL, + [_BINARY_OP_SUBSCR_INIT_CALL_r11] = _BINARY_OP_SUBSCR_INIT_CALL, + [_BINARY_OP_SUBSCR_INIT_CALL_r21] = _BINARY_OP_SUBSCR_INIT_CALL, + [_BINARY_OP_SUBSCR_INIT_CALL_r31] = _BINARY_OP_SUBSCR_INIT_CALL, + [_LIST_APPEND_r10] = _LIST_APPEND, + [_SET_ADD_r10] = _SET_ADD, + [_STORE_SUBSCR_r30] = _STORE_SUBSCR, + [_STORE_SUBSCR_LIST_INT_r32] = _STORE_SUBSCR_LIST_INT, + [_STORE_SUBSCR_DICT_r31] = _STORE_SUBSCR_DICT, + [_STORE_SUBSCR_DICT_KNOWN_HASH_r31] = _STORE_SUBSCR_DICT_KNOWN_HASH, + [_DELETE_SUBSCR_r20] = _DELETE_SUBSCR, + [_CALL_INTRINSIC_1_r12] = _CALL_INTRINSIC_1, + [_CALL_INTRINSIC_2_r23] = _CALL_INTRINSIC_2, + [_MAKE_HEAP_SAFE_r01] = _MAKE_HEAP_SAFE, + [_MAKE_HEAP_SAFE_r11] = _MAKE_HEAP_SAFE, + [_MAKE_HEAP_SAFE_r22] = _MAKE_HEAP_SAFE, + [_MAKE_HEAP_SAFE_r33] = _MAKE_HEAP_SAFE, + [_RETURN_VALUE_r11] = _RETURN_VALUE, + [_GET_AITER_r11] = _GET_AITER, + [_GET_ANEXT_r12] = _GET_ANEXT, + [_GET_AWAITABLE_r11] = _GET_AWAITABLE, + [_SEND_GEN_FRAME_r33] = _SEND_GEN_FRAME, + [_GUARD_TOS_IS_NONE_r01] = _GUARD_TOS_IS_NONE, + [_GUARD_TOS_IS_NONE_r11] = _GUARD_TOS_IS_NONE, + [_GUARD_TOS_IS_NONE_r22] = _GUARD_TOS_IS_NONE, + [_GUARD_TOS_IS_NONE_r33] = _GUARD_TOS_IS_NONE, + [_GUARD_NOS_NOT_NULL_r02] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r12] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r22] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r33] = _GUARD_NOS_NOT_NULL, + [_SEND_VIRTUAL_TIER_TWO_r03] = _SEND_VIRTUAL_TIER_TWO, + [_SEND_VIRTUAL_TIER_TWO_r13] = _SEND_VIRTUAL_TIER_TWO, + [_SEND_VIRTUAL_TIER_TWO_r23] = _SEND_VIRTUAL_TIER_TWO, + [_SEND_VIRTUAL_TIER_TWO_r33] = _SEND_VIRTUAL_TIER_TWO, + [_GUARD_3OS_ASYNC_GEN_ASEND_r03] = _GUARD_3OS_ASYNC_GEN_ASEND, + [_GUARD_3OS_ASYNC_GEN_ASEND_r13] = _GUARD_3OS_ASYNC_GEN_ASEND, + [_GUARD_3OS_ASYNC_GEN_ASEND_r23] = _GUARD_3OS_ASYNC_GEN_ASEND, + [_GUARD_3OS_ASYNC_GEN_ASEND_r33] = _GUARD_3OS_ASYNC_GEN_ASEND, + [_SEND_ASYNC_GEN_TIER_TWO_r33] = _SEND_ASYNC_GEN_TIER_TWO, + [_YIELD_VALUE_r11] = _YIELD_VALUE, + [_POP_EXCEPT_r10] = _POP_EXCEPT, + [_LOAD_COMMON_CONSTANT_r01] = _LOAD_COMMON_CONSTANT, + [_LOAD_COMMON_CONSTANT_r12] = _LOAD_COMMON_CONSTANT, + [_LOAD_COMMON_CONSTANT_r23] = _LOAD_COMMON_CONSTANT, + [_LOAD_BUILD_CLASS_r01] = _LOAD_BUILD_CLASS, + [_STORE_NAME_r10] = _STORE_NAME, + [_DELETE_NAME_r00] = _DELETE_NAME, + [_UNPACK_SEQUENCE_r10] = _UNPACK_SEQUENCE, + [_UNPACK_SEQUENCE_TWO_TUPLE_r12] = _UNPACK_SEQUENCE_TWO_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02] = _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12] = _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23] = _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03] = _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13] = _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE, + [_UNPACK_SEQUENCE_TUPLE_r10] = _UNPACK_SEQUENCE_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_TUPLE_r10] = _UNPACK_SEQUENCE_UNIQUE_TUPLE, + [_UNPACK_SEQUENCE_LIST_r10] = _UNPACK_SEQUENCE_LIST, + [_UNPACK_EX_r10] = _UNPACK_EX, + [_STORE_ATTR_r20] = _STORE_ATTR, + [_DELETE_ATTR_r10] = _DELETE_ATTR, + [_STORE_GLOBAL_r10] = _STORE_GLOBAL, + [_DELETE_GLOBAL_r00] = _DELETE_GLOBAL, + [_LOAD_LOCALS_r01] = _LOAD_LOCALS, + [_LOAD_LOCALS_r12] = _LOAD_LOCALS, + [_LOAD_LOCALS_r23] = _LOAD_LOCALS, + [_LOAD_NAME_r01] = _LOAD_NAME, + [_LOAD_GLOBAL_r00] = _LOAD_GLOBAL, + [_PUSH_NULL_CONDITIONAL_r00] = _PUSH_NULL_CONDITIONAL, + [_GUARD_GLOBALS_VERSION_r00] = _GUARD_GLOBALS_VERSION, + [_GUARD_GLOBALS_VERSION_r11] = _GUARD_GLOBALS_VERSION, + [_GUARD_GLOBALS_VERSION_r22] = _GUARD_GLOBALS_VERSION, + [_GUARD_GLOBALS_VERSION_r33] = _GUARD_GLOBALS_VERSION, + [_LOAD_GLOBAL_MODULE_r01] = _LOAD_GLOBAL_MODULE, + [_LOAD_GLOBAL_BUILTINS_r01] = _LOAD_GLOBAL_BUILTINS, + [_DELETE_FAST_r00] = _DELETE_FAST, + [_MAKE_CELL_r00] = _MAKE_CELL, + [_DELETE_DEREF_r00] = _DELETE_DEREF, + [_LOAD_FROM_DICT_OR_DEREF_r11] = _LOAD_FROM_DICT_OR_DEREF, + [_LOAD_DEREF_r01] = _LOAD_DEREF, + [_STORE_DEREF_r10] = _STORE_DEREF, + [_COPY_FREE_VARS_r00] = _COPY_FREE_VARS, + [_COPY_FREE_VARS_r11] = _COPY_FREE_VARS, + [_COPY_FREE_VARS_r22] = _COPY_FREE_VARS, + [_COPY_FREE_VARS_r33] = _COPY_FREE_VARS, + [_BUILD_STRING_r01] = _BUILD_STRING, + [_BUILD_INTERPOLATION_r01] = _BUILD_INTERPOLATION, + [_BUILD_TEMPLATE_r21] = _BUILD_TEMPLATE, + [_BUILD_TUPLE_r01] = _BUILD_TUPLE, + [_BUILD_LIST_r01] = _BUILD_LIST, + [_LIST_EXTEND_r11] = _LIST_EXTEND, + [_SET_UPDATE_r11] = _SET_UPDATE, + [_BUILD_SET_r01] = _BUILD_SET, + [_BUILD_MAP_r01] = _BUILD_MAP, + [_SETUP_ANNOTATIONS_r00] = _SETUP_ANNOTATIONS, + [_DICT_UPDATE_r11] = _DICT_UPDATE, + [_DICT_MERGE_r11] = _DICT_MERGE, + [_MAP_ADD_r20] = _MAP_ADD, + [_LOAD_SUPER_ATTR_ATTR_r31] = _LOAD_SUPER_ATTR_ATTR, + [_GUARD_NOS_TYPE_VERSION_r02] = _GUARD_NOS_TYPE_VERSION, + [_GUARD_NOS_TYPE_VERSION_r12] = _GUARD_NOS_TYPE_VERSION, + [_GUARD_NOS_TYPE_VERSION_r22] = _GUARD_NOS_TYPE_VERSION, + [_GUARD_NOS_TYPE_VERSION_r33] = _GUARD_NOS_TYPE_VERSION, + [_GUARD_LOAD_SUPER_ATTR_METHOD_r03] = _GUARD_LOAD_SUPER_ATTR_METHOD, + [_GUARD_LOAD_SUPER_ATTR_METHOD_r13] = _GUARD_LOAD_SUPER_ATTR_METHOD, + [_GUARD_LOAD_SUPER_ATTR_METHOD_r23] = _GUARD_LOAD_SUPER_ATTR_METHOD, + [_GUARD_LOAD_SUPER_ATTR_METHOD_r33] = _GUARD_LOAD_SUPER_ATTR_METHOD, + [_LOAD_SUPER_ATTR_METHOD_r32] = _LOAD_SUPER_ATTR_METHOD, + [_LOAD_ATTR_r10] = _LOAD_ATTR, + [_GUARD_TYPE_VERSION_r01] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_r11] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_r22] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_r33] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_LOCKED_r01] = _GUARD_TYPE_VERSION_LOCKED, + [_GUARD_TYPE_VERSION_LOCKED_r11] = _GUARD_TYPE_VERSION_LOCKED, + [_GUARD_TYPE_VERSION_LOCKED_r22] = _GUARD_TYPE_VERSION_LOCKED, + [_GUARD_TYPE_VERSION_LOCKED_r33] = _GUARD_TYPE_VERSION_LOCKED, + [_GUARD_TYPE_r01] = _GUARD_TYPE, + [_GUARD_TYPE_r11] = _GUARD_TYPE, + [_GUARD_TYPE_r22] = _GUARD_TYPE, + [_GUARD_TYPE_r33] = _GUARD_TYPE, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r01] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r11] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r22] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r33] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_LOAD_ATTR_INSTANCE_VALUE_r02] = _LOAD_ATTR_INSTANCE_VALUE, + [_LOAD_ATTR_INSTANCE_VALUE_r12] = _LOAD_ATTR_INSTANCE_VALUE, + [_LOAD_ATTR_INSTANCE_VALUE_r23] = _LOAD_ATTR_INSTANCE_VALUE, + [_LOAD_ATTR_MODULE_r12] = _LOAD_ATTR_MODULE, + [_LOAD_ATTR_WITH_HINT_r12] = _LOAD_ATTR_WITH_HINT, + [_LOAD_ATTR_SLOT_r02] = _LOAD_ATTR_SLOT, + [_LOAD_ATTR_SLOT_r12] = _LOAD_ATTR_SLOT, + [_LOAD_ATTR_SLOT_r23] = _LOAD_ATTR_SLOT, + [_CHECK_ATTR_CLASS_r01] = _CHECK_ATTR_CLASS, + [_CHECK_ATTR_CLASS_r11] = _CHECK_ATTR_CLASS, + [_CHECK_ATTR_CLASS_r22] = _CHECK_ATTR_CLASS, + [_CHECK_ATTR_CLASS_r33] = _CHECK_ATTR_CLASS, + [_LOAD_ATTR_CLASS_r11] = _LOAD_ATTR_CLASS, + [_LOAD_ATTR_PROPERTY_FRAME_r01] = _LOAD_ATTR_PROPERTY_FRAME, + [_LOAD_ATTR_PROPERTY_FRAME_r11] = _LOAD_ATTR_PROPERTY_FRAME, + [_LOAD_ATTR_PROPERTY_FRAME_r22] = _LOAD_ATTR_PROPERTY_FRAME, + [_LOAD_ATTR_PROPERTY_FRAME_r33] = _LOAD_ATTR_PROPERTY_FRAME, + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11] = _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME, + [_GUARD_DORV_NO_DICT_r01] = _GUARD_DORV_NO_DICT, + [_GUARD_DORV_NO_DICT_r11] = _GUARD_DORV_NO_DICT, + [_GUARD_DORV_NO_DICT_r22] = _GUARD_DORV_NO_DICT, + [_GUARD_DORV_NO_DICT_r33] = _GUARD_DORV_NO_DICT, + [_STORE_ATTR_INSTANCE_VALUE_r21] = _STORE_ATTR_INSTANCE_VALUE, + [_LOCK_OBJECT_r01] = _LOCK_OBJECT, + [_LOCK_OBJECT_r11] = _LOCK_OBJECT, + [_LOCK_OBJECT_r22] = _LOCK_OBJECT, + [_LOCK_OBJECT_r33] = _LOCK_OBJECT, + [_STORE_ATTR_WITH_HINT_r21] = _STORE_ATTR_WITH_HINT, + [_STORE_ATTR_SLOT_r21] = _STORE_ATTR_SLOT, + [_COMPARE_OP_r21] = _COMPARE_OP, + [_COMPARE_OP_FLOAT_r03] = _COMPARE_OP_FLOAT, + [_COMPARE_OP_FLOAT_r13] = _COMPARE_OP_FLOAT, + [_COMPARE_OP_FLOAT_r23] = _COMPARE_OP_FLOAT, + [_COMPARE_OP_INT_r23] = _COMPARE_OP_INT, + [_COMPARE_OP_STR_r23] = _COMPARE_OP_STR, + [_IS_OP_r03] = _IS_OP, + [_IS_OP_r13] = _IS_OP, + [_IS_OP_r23] = _IS_OP, + [_CONTAINS_OP_r23] = _CONTAINS_OP, + [_GUARD_TOS_ANY_SET_r01] = _GUARD_TOS_ANY_SET, + [_GUARD_TOS_ANY_SET_r11] = _GUARD_TOS_ANY_SET, + [_GUARD_TOS_ANY_SET_r22] = _GUARD_TOS_ANY_SET, + [_GUARD_TOS_ANY_SET_r33] = _GUARD_TOS_ANY_SET, + [_GUARD_TOS_SET_r01] = _GUARD_TOS_SET, + [_GUARD_TOS_SET_r11] = _GUARD_TOS_SET, + [_GUARD_TOS_SET_r22] = _GUARD_TOS_SET, + [_GUARD_TOS_SET_r33] = _GUARD_TOS_SET, + [_GUARD_TOS_FROZENSET_r01] = _GUARD_TOS_FROZENSET, + [_GUARD_TOS_FROZENSET_r11] = _GUARD_TOS_FROZENSET, + [_GUARD_TOS_FROZENSET_r22] = _GUARD_TOS_FROZENSET, + [_GUARD_TOS_FROZENSET_r33] = _GUARD_TOS_FROZENSET, + [_CONTAINS_OP_SET_r23] = _CONTAINS_OP_SET, + [_CONTAINS_OP_DICT_r23] = _CONTAINS_OP_DICT, + [_CHECK_EG_MATCH_r22] = _CHECK_EG_MATCH, + [_CHECK_EXC_MATCH_r22] = _CHECK_EXC_MATCH, + [_IMPORT_NAME_r21] = _IMPORT_NAME, + [_IMPORT_FROM_r12] = _IMPORT_FROM, + [_IS_NONE_r11] = _IS_NONE, + [_GET_LEN_r12] = _GET_LEN, + [_MATCH_CLASS_r33] = _MATCH_CLASS, + [_MATCH_MAPPING_r02] = _MATCH_MAPPING, + [_MATCH_MAPPING_r12] = _MATCH_MAPPING, + [_MATCH_MAPPING_r23] = _MATCH_MAPPING, + [_MATCH_SEQUENCE_r02] = _MATCH_SEQUENCE, + [_MATCH_SEQUENCE_r12] = _MATCH_SEQUENCE, + [_MATCH_SEQUENCE_r23] = _MATCH_SEQUENCE, + [_MATCH_KEYS_r23] = _MATCH_KEYS, + [_GET_ITER_r12] = _GET_ITER, + [_GUARD_ITERATOR_r01] = _GUARD_ITERATOR, + [_GUARD_ITERATOR_r11] = _GUARD_ITERATOR, + [_GUARD_ITERATOR_r22] = _GUARD_ITERATOR, + [_GUARD_ITERATOR_r33] = _GUARD_ITERATOR, + [_GUARD_ITER_VIRTUAL_r01] = _GUARD_ITER_VIRTUAL, + [_GUARD_ITER_VIRTUAL_r11] = _GUARD_ITER_VIRTUAL, + [_GUARD_ITER_VIRTUAL_r22] = _GUARD_ITER_VIRTUAL, + [_GUARD_ITER_VIRTUAL_r33] = _GUARD_ITER_VIRTUAL, + [_PUSH_TAGGED_ZERO_r01] = _PUSH_TAGGED_ZERO, + [_PUSH_TAGGED_ZERO_r12] = _PUSH_TAGGED_ZERO, + [_PUSH_TAGGED_ZERO_r23] = _PUSH_TAGGED_ZERO, + [_GET_ITER_TRAD_r12] = _GET_ITER_TRAD, + [_FOR_ITER_TIER_TWO_r23] = _FOR_ITER_TIER_TWO, + [_GUARD_TYPE_ITER_r02] = _GUARD_TYPE_ITER, + [_GUARD_TYPE_ITER_r12] = _GUARD_TYPE_ITER, + [_GUARD_TYPE_ITER_r22] = _GUARD_TYPE_ITER, + [_GUARD_TYPE_ITER_r33] = _GUARD_TYPE_ITER, + [_ITER_NEXT_INLINE_r23] = _ITER_NEXT_INLINE, + [_GUARD_NOS_ITER_VIRTUAL_r02] = _GUARD_NOS_ITER_VIRTUAL, + [_GUARD_NOS_ITER_VIRTUAL_r12] = _GUARD_NOS_ITER_VIRTUAL, + [_GUARD_NOS_ITER_VIRTUAL_r22] = _GUARD_NOS_ITER_VIRTUAL, + [_GUARD_NOS_ITER_VIRTUAL_r33] = _GUARD_NOS_ITER_VIRTUAL, + [_GUARD_TOS_NOT_NULL_r01] = _GUARD_TOS_NOT_NULL, + [_GUARD_TOS_NOT_NULL_r11] = _GUARD_TOS_NOT_NULL, + [_GUARD_TOS_NOT_NULL_r22] = _GUARD_TOS_NOT_NULL, + [_GUARD_TOS_NOT_NULL_r33] = _GUARD_TOS_NOT_NULL, + [_FOR_ITER_VIRTUAL_TIER_TWO_r23] = _FOR_ITER_VIRTUAL_TIER_TWO, + [_ITER_CHECK_LIST_r02] = _ITER_CHECK_LIST, + [_ITER_CHECK_LIST_r12] = _ITER_CHECK_LIST, + [_ITER_CHECK_LIST_r22] = _ITER_CHECK_LIST, + [_ITER_CHECK_LIST_r33] = _ITER_CHECK_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r02] = _GUARD_NOT_EXHAUSTED_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r12] = _GUARD_NOT_EXHAUSTED_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r22] = _GUARD_NOT_EXHAUSTED_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r33] = _GUARD_NOT_EXHAUSTED_LIST, + [_ITER_NEXT_LIST_TIER_TWO_r23] = _ITER_NEXT_LIST_TIER_TWO, + [_ITER_CHECK_TUPLE_r02] = _ITER_CHECK_TUPLE, + [_ITER_CHECK_TUPLE_r12] = _ITER_CHECK_TUPLE, + [_ITER_CHECK_TUPLE_r22] = _ITER_CHECK_TUPLE, + [_ITER_CHECK_TUPLE_r33] = _ITER_CHECK_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r02] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r12] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r22] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r33] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_ITER_NEXT_TUPLE_r03] = _ITER_NEXT_TUPLE, + [_ITER_NEXT_TUPLE_r13] = _ITER_NEXT_TUPLE, + [_ITER_NEXT_TUPLE_r23] = _ITER_NEXT_TUPLE, + [_ITER_CHECK_RANGE_r02] = _ITER_CHECK_RANGE, + [_ITER_CHECK_RANGE_r12] = _ITER_CHECK_RANGE, + [_ITER_CHECK_RANGE_r22] = _ITER_CHECK_RANGE, + [_ITER_CHECK_RANGE_r33] = _ITER_CHECK_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r02] = _GUARD_NOT_EXHAUSTED_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r12] = _GUARD_NOT_EXHAUSTED_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r22] = _GUARD_NOT_EXHAUSTED_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r33] = _GUARD_NOT_EXHAUSTED_RANGE, + [_ITER_NEXT_RANGE_r03] = _ITER_NEXT_RANGE, + [_ITER_NEXT_RANGE_r13] = _ITER_NEXT_RANGE, + [_ITER_NEXT_RANGE_r23] = _ITER_NEXT_RANGE, + [_FOR_ITER_GEN_FRAME_r03] = _FOR_ITER_GEN_FRAME, + [_FOR_ITER_GEN_FRAME_r13] = _FOR_ITER_GEN_FRAME, + [_FOR_ITER_GEN_FRAME_r23] = _FOR_ITER_GEN_FRAME, + [_INSERT_NULL_r10] = _INSERT_NULL, + [_LOAD_SPECIAL_r00] = _LOAD_SPECIAL, + [_WITH_EXCEPT_START_r33] = _WITH_EXCEPT_START, + [_PUSH_EXC_INFO_r02] = _PUSH_EXC_INFO, + [_PUSH_EXC_INFO_r12] = _PUSH_EXC_INFO, + [_PUSH_EXC_INFO_r23] = _PUSH_EXC_INFO, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, + [_LOAD_ATTR_METHOD_WITH_VALUES_r02] = _LOAD_ATTR_METHOD_WITH_VALUES, + [_LOAD_ATTR_METHOD_WITH_VALUES_r12] = _LOAD_ATTR_METHOD_WITH_VALUES, + [_LOAD_ATTR_METHOD_WITH_VALUES_r23] = _LOAD_ATTR_METHOD_WITH_VALUES, + [_LOAD_ATTR_METHOD_NO_DICT_r02] = _LOAD_ATTR_METHOD_NO_DICT, + [_LOAD_ATTR_METHOD_NO_DICT_r12] = _LOAD_ATTR_METHOD_NO_DICT, + [_LOAD_ATTR_METHOD_NO_DICT_r23] = _LOAD_ATTR_METHOD_NO_DICT, + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11] = _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11] = _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r01] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r11] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r22] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r33] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_LOAD_ATTR_METHOD_LAZY_DICT_r02] = _LOAD_ATTR_METHOD_LAZY_DICT, + [_LOAD_ATTR_METHOD_LAZY_DICT_r12] = _LOAD_ATTR_METHOD_LAZY_DICT, + [_LOAD_ATTR_METHOD_LAZY_DICT_r23] = _LOAD_ATTR_METHOD_LAZY_DICT, + [_MAYBE_EXPAND_METHOD_r00] = _MAYBE_EXPAND_METHOD, + [_PY_FRAME_GENERAL_r01] = _PY_FRAME_GENERAL, + [_CHECK_FUNCTION_VERSION_r00] = _CHECK_FUNCTION_VERSION, + [_CHECK_FUNCTION_VERSION_INLINE_r00] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_FUNCTION_VERSION_INLINE_r11] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_FUNCTION_VERSION_INLINE_r22] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_FUNCTION_VERSION_INLINE_r33] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_METHOD_VERSION_r00] = _CHECK_METHOD_VERSION, + [_EXPAND_METHOD_r00] = _EXPAND_METHOD, + [_CHECK_IS_NOT_PY_CALLABLE_r00] = _CHECK_IS_NOT_PY_CALLABLE, + [_CALL_NON_PY_GENERAL_r01] = _CALL_NON_PY_GENERAL, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00] = _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00] = _INIT_CALL_BOUND_METHOD_EXACT_ARGS, + [_CHECK_PEP_523_r00] = _CHECK_PEP_523, + [_CHECK_PEP_523_r11] = _CHECK_PEP_523, + [_CHECK_PEP_523_r22] = _CHECK_PEP_523, + [_CHECK_PEP_523_r33] = _CHECK_PEP_523, + [_CHECK_FUNCTION_EXACT_ARGS_r00] = _CHECK_FUNCTION_EXACT_ARGS, + [_CHECK_STACK_SPACE_r00] = _CHECK_STACK_SPACE, + [_CHECK_RECURSION_REMAINING_r00] = _CHECK_RECURSION_REMAINING, + [_CHECK_RECURSION_REMAINING_r11] = _CHECK_RECURSION_REMAINING, + [_CHECK_RECURSION_REMAINING_r22] = _CHECK_RECURSION_REMAINING, + [_CHECK_RECURSION_REMAINING_r33] = _CHECK_RECURSION_REMAINING, + [_INIT_CALL_PY_EXACT_ARGS_0_r01] = _INIT_CALL_PY_EXACT_ARGS_0, + [_INIT_CALL_PY_EXACT_ARGS_1_r01] = _INIT_CALL_PY_EXACT_ARGS_1, + [_INIT_CALL_PY_EXACT_ARGS_2_r01] = _INIT_CALL_PY_EXACT_ARGS_2, + [_INIT_CALL_PY_EXACT_ARGS_3_r01] = _INIT_CALL_PY_EXACT_ARGS_3, + [_INIT_CALL_PY_EXACT_ARGS_4_r01] = _INIT_CALL_PY_EXACT_ARGS_4, + [_INIT_CALL_PY_EXACT_ARGS_r01] = _INIT_CALL_PY_EXACT_ARGS, + [_PUSH_FRAME_r10] = _PUSH_FRAME, + [_GUARD_NOS_NULL_r02] = _GUARD_NOS_NULL, + [_GUARD_NOS_NULL_r12] = _GUARD_NOS_NULL, + [_GUARD_NOS_NULL_r22] = _GUARD_NOS_NULL, + [_GUARD_NOS_NULL_r33] = _GUARD_NOS_NULL, + [_GUARD_THIRD_NULL_r03] = _GUARD_THIRD_NULL, + [_GUARD_THIRD_NULL_r13] = _GUARD_THIRD_NULL, + [_GUARD_THIRD_NULL_r23] = _GUARD_THIRD_NULL, + [_GUARD_THIRD_NULL_r33] = _GUARD_THIRD_NULL, + [_GUARD_CALLABLE_TYPE_1_r03] = _GUARD_CALLABLE_TYPE_1, + [_GUARD_CALLABLE_TYPE_1_r13] = _GUARD_CALLABLE_TYPE_1, + [_GUARD_CALLABLE_TYPE_1_r23] = _GUARD_CALLABLE_TYPE_1, + [_GUARD_CALLABLE_TYPE_1_r33] = _GUARD_CALLABLE_TYPE_1, + [_CALL_TYPE_1_r02] = _CALL_TYPE_1, + [_CALL_TYPE_1_r12] = _CALL_TYPE_1, + [_CALL_TYPE_1_r22] = _CALL_TYPE_1, + [_CALL_TYPE_1_r32] = _CALL_TYPE_1, + [_GUARD_CALLABLE_STR_1_r03] = _GUARD_CALLABLE_STR_1, + [_GUARD_CALLABLE_STR_1_r13] = _GUARD_CALLABLE_STR_1, + [_GUARD_CALLABLE_STR_1_r23] = _GUARD_CALLABLE_STR_1, + [_GUARD_CALLABLE_STR_1_r33] = _GUARD_CALLABLE_STR_1, + [_CALL_STR_1_r32] = _CALL_STR_1, + [_GUARD_CALLABLE_TUPLE_1_r03] = _GUARD_CALLABLE_TUPLE_1, + [_GUARD_CALLABLE_TUPLE_1_r13] = _GUARD_CALLABLE_TUPLE_1, + [_GUARD_CALLABLE_TUPLE_1_r23] = _GUARD_CALLABLE_TUPLE_1, + [_GUARD_CALLABLE_TUPLE_1_r33] = _GUARD_CALLABLE_TUPLE_1, + [_CALL_TUPLE_1_r32] = _CALL_TUPLE_1, + [_CHECK_OBJECT_r00] = _CHECK_OBJECT, + [_ALLOCATE_OBJECT_r00] = _ALLOCATE_OBJECT, + [_CREATE_INIT_FRAME_r01] = _CREATE_INIT_FRAME, + [_EXIT_INIT_CHECK_r10] = _EXIT_INIT_CHECK, + [_GUARD_CALLABLE_BUILTIN_CLASS_r00] = _GUARD_CALLABLE_BUILTIN_CLASS, + [_CALL_BUILTIN_CLASS_r00] = _CALL_BUILTIN_CLASS, + [_GUARD_CALLABLE_BUILTIN_O_r00] = _GUARD_CALLABLE_BUILTIN_O, + [_CALL_BUILTIN_O_r03] = _CALL_BUILTIN_O, + [_GUARD_CALLABLE_BUILTIN_FAST_r00] = _GUARD_CALLABLE_BUILTIN_FAST, + [_CALL_BUILTIN_FAST_r00] = _CALL_BUILTIN_FAST, + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00] = _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS_r00] = _CALL_BUILTIN_FAST_WITH_KEYWORDS, + [_GUARD_CALLABLE_LEN_r03] = _GUARD_CALLABLE_LEN, + [_GUARD_CALLABLE_LEN_r13] = _GUARD_CALLABLE_LEN, + [_GUARD_CALLABLE_LEN_r23] = _GUARD_CALLABLE_LEN, + [_GUARD_CALLABLE_LEN_r33] = _GUARD_CALLABLE_LEN, + [_CALL_LEN_r33] = _CALL_LEN, + [_GUARD_CALLABLE_ISINSTANCE_r03] = _GUARD_CALLABLE_ISINSTANCE, + [_GUARD_CALLABLE_ISINSTANCE_r13] = _GUARD_CALLABLE_ISINSTANCE, + [_GUARD_CALLABLE_ISINSTANCE_r23] = _GUARD_CALLABLE_ISINSTANCE, + [_GUARD_CALLABLE_ISINSTANCE_r33] = _GUARD_CALLABLE_ISINSTANCE, + [_CALL_ISINSTANCE_r31] = _CALL_ISINSTANCE, + [_GUARD_CALLABLE_LIST_APPEND_r03] = _GUARD_CALLABLE_LIST_APPEND, + [_GUARD_CALLABLE_LIST_APPEND_r13] = _GUARD_CALLABLE_LIST_APPEND, + [_GUARD_CALLABLE_LIST_APPEND_r23] = _GUARD_CALLABLE_LIST_APPEND, + [_GUARD_CALLABLE_LIST_APPEND_r33] = _GUARD_CALLABLE_LIST_APPEND, + [_CALL_LIST_APPEND_r03] = _CALL_LIST_APPEND, + [_CALL_LIST_APPEND_r13] = _CALL_LIST_APPEND, + [_CALL_LIST_APPEND_r23] = _CALL_LIST_APPEND, + [_CALL_LIST_APPEND_r33] = _CALL_LIST_APPEND, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_O, + [_CALL_METHOD_DESCRIPTOR_O_r03] = _CALL_METHOD_DESCRIPTOR_O, + [_CHECK_RECURSION_LIMIT_r00] = _CHECK_RECURSION_LIMIT, + [_CHECK_RECURSION_LIMIT_r11] = _CHECK_RECURSION_LIMIT, + [_CHECK_RECURSION_LIMIT_r22] = _CHECK_RECURSION_LIMIT, + [_CHECK_RECURSION_LIMIT_r33] = _CHECK_RECURSION_LIMIT, + [_CALL_METHOD_DESCRIPTOR_O_INLINE_r03] = _CALL_METHOD_DESCRIPTOR_O_INLINE, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, + [_CALL_METHOD_DESCRIPTOR_NOARGS_r03] = _CALL_METHOD_DESCRIPTOR_NOARGS, + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03] = _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, + [_CALL_METHOD_DESCRIPTOR_FAST_r00] = _CALL_METHOD_DESCRIPTOR_FAST, + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00] = _CALL_METHOD_DESCRIPTOR_FAST_INLINE, + [_MAYBE_EXPAND_METHOD_KW_r11] = _MAYBE_EXPAND_METHOD_KW, + [_PY_FRAME_KW_r11] = _PY_FRAME_KW, + [_CHECK_FUNCTION_VERSION_KW_r11] = _CHECK_FUNCTION_VERSION_KW, + [_CHECK_METHOD_VERSION_KW_r11] = _CHECK_METHOD_VERSION_KW, + [_EXPAND_METHOD_KW_r11] = _EXPAND_METHOD_KW, + [_CHECK_IS_NOT_PY_CALLABLE_KW_r11] = _CHECK_IS_NOT_PY_CALLABLE_KW, + [_CALL_KW_NON_PY_r11] = _CALL_KW_NON_PY, + [_MAKE_CALLARGS_A_TUPLE_r33] = _MAKE_CALLARGS_A_TUPLE, + [_CHECK_IS_PY_CALLABLE_EX_r03] = _CHECK_IS_PY_CALLABLE_EX, + [_CHECK_IS_PY_CALLABLE_EX_r13] = _CHECK_IS_PY_CALLABLE_EX, + [_CHECK_IS_PY_CALLABLE_EX_r23] = _CHECK_IS_PY_CALLABLE_EX, + [_CHECK_IS_PY_CALLABLE_EX_r33] = _CHECK_IS_PY_CALLABLE_EX, + [_PY_FRAME_EX_r31] = _PY_FRAME_EX, + [_CHECK_IS_NOT_PY_CALLABLE_EX_r03] = _CHECK_IS_NOT_PY_CALLABLE_EX, + [_CHECK_IS_NOT_PY_CALLABLE_EX_r13] = _CHECK_IS_NOT_PY_CALLABLE_EX, + [_CHECK_IS_NOT_PY_CALLABLE_EX_r23] = _CHECK_IS_NOT_PY_CALLABLE_EX, + [_CHECK_IS_NOT_PY_CALLABLE_EX_r33] = _CHECK_IS_NOT_PY_CALLABLE_EX, + [_CALL_FUNCTION_EX_NON_PY_GENERAL_r31] = _CALL_FUNCTION_EX_NON_PY_GENERAL, + [_MAKE_FUNCTION_r12] = _MAKE_FUNCTION, + [_SET_FUNCTION_ATTRIBUTE_r01] = _SET_FUNCTION_ATTRIBUTE, + [_SET_FUNCTION_ATTRIBUTE_r11] = _SET_FUNCTION_ATTRIBUTE, + [_SET_FUNCTION_ATTRIBUTE_r21] = _SET_FUNCTION_ATTRIBUTE, + [_SET_FUNCTION_ATTRIBUTE_r32] = _SET_FUNCTION_ATTRIBUTE, + [_RETURN_GENERATOR_r01] = _RETURN_GENERATOR, + [_BUILD_SLICE_r01] = _BUILD_SLICE, + [_CONVERT_VALUE_r11] = _CONVERT_VALUE, + [_FORMAT_SIMPLE_r11] = _FORMAT_SIMPLE, + [_FORMAT_WITH_SPEC_r21] = _FORMAT_WITH_SPEC, + [_COPY_1_r02] = _COPY_1, + [_COPY_1_r12] = _COPY_1, + [_COPY_1_r23] = _COPY_1, + [_COPY_2_r03] = _COPY_2, + [_COPY_2_r13] = _COPY_2, + [_COPY_2_r23] = _COPY_2, + [_COPY_3_r03] = _COPY_3, + [_COPY_3_r13] = _COPY_3, + [_COPY_3_r23] = _COPY_3, + [_COPY_3_r33] = _COPY_3, + [_COPY_r01] = _COPY, + [_BINARY_OP_r23] = _BINARY_OP, + [_SWAP_2_r02] = _SWAP_2, + [_SWAP_2_r12] = _SWAP_2, + [_SWAP_2_r22] = _SWAP_2, + [_SWAP_2_r33] = _SWAP_2, + [_SWAP_3_r03] = _SWAP_3, + [_SWAP_3_r13] = _SWAP_3, + [_SWAP_3_r23] = _SWAP_3, + [_SWAP_3_r33] = _SWAP_3, + [_SWAP_r11] = _SWAP, + [_GUARD_IS_TRUE_POP_r00] = _GUARD_IS_TRUE_POP, + [_GUARD_IS_TRUE_POP_r10] = _GUARD_IS_TRUE_POP, + [_GUARD_IS_TRUE_POP_r21] = _GUARD_IS_TRUE_POP, + [_GUARD_IS_TRUE_POP_r32] = _GUARD_IS_TRUE_POP, + [_GUARD_IS_FALSE_POP_r00] = _GUARD_IS_FALSE_POP, + [_GUARD_IS_FALSE_POP_r10] = _GUARD_IS_FALSE_POP, + [_GUARD_IS_FALSE_POP_r21] = _GUARD_IS_FALSE_POP, + [_GUARD_IS_FALSE_POP_r32] = _GUARD_IS_FALSE_POP, + [_GUARD_BIT_IS_SET_POP_4_r00] = _GUARD_BIT_IS_SET_POP_4, + [_GUARD_BIT_IS_SET_POP_4_r10] = _GUARD_BIT_IS_SET_POP_4, + [_GUARD_BIT_IS_SET_POP_4_r21] = _GUARD_BIT_IS_SET_POP_4, + [_GUARD_BIT_IS_SET_POP_4_r32] = _GUARD_BIT_IS_SET_POP_4, + [_GUARD_BIT_IS_SET_POP_5_r00] = _GUARD_BIT_IS_SET_POP_5, + [_GUARD_BIT_IS_SET_POP_5_r10] = _GUARD_BIT_IS_SET_POP_5, + [_GUARD_BIT_IS_SET_POP_5_r21] = _GUARD_BIT_IS_SET_POP_5, + [_GUARD_BIT_IS_SET_POP_5_r32] = _GUARD_BIT_IS_SET_POP_5, + [_GUARD_BIT_IS_SET_POP_6_r00] = _GUARD_BIT_IS_SET_POP_6, + [_GUARD_BIT_IS_SET_POP_6_r10] = _GUARD_BIT_IS_SET_POP_6, + [_GUARD_BIT_IS_SET_POP_6_r21] = _GUARD_BIT_IS_SET_POP_6, + [_GUARD_BIT_IS_SET_POP_6_r32] = _GUARD_BIT_IS_SET_POP_6, + [_GUARD_BIT_IS_SET_POP_7_r00] = _GUARD_BIT_IS_SET_POP_7, + [_GUARD_BIT_IS_SET_POP_7_r10] = _GUARD_BIT_IS_SET_POP_7, + [_GUARD_BIT_IS_SET_POP_7_r21] = _GUARD_BIT_IS_SET_POP_7, + [_GUARD_BIT_IS_SET_POP_7_r32] = _GUARD_BIT_IS_SET_POP_7, + [_GUARD_BIT_IS_SET_POP_r00] = _GUARD_BIT_IS_SET_POP, + [_GUARD_BIT_IS_SET_POP_r10] = _GUARD_BIT_IS_SET_POP, + [_GUARD_BIT_IS_SET_POP_r21] = _GUARD_BIT_IS_SET_POP, + [_GUARD_BIT_IS_SET_POP_r32] = _GUARD_BIT_IS_SET_POP, + [_GUARD_BIT_IS_UNSET_POP_4_r00] = _GUARD_BIT_IS_UNSET_POP_4, + [_GUARD_BIT_IS_UNSET_POP_4_r10] = _GUARD_BIT_IS_UNSET_POP_4, + [_GUARD_BIT_IS_UNSET_POP_4_r21] = _GUARD_BIT_IS_UNSET_POP_4, + [_GUARD_BIT_IS_UNSET_POP_4_r32] = _GUARD_BIT_IS_UNSET_POP_4, + [_GUARD_BIT_IS_UNSET_POP_5_r00] = _GUARD_BIT_IS_UNSET_POP_5, + [_GUARD_BIT_IS_UNSET_POP_5_r10] = _GUARD_BIT_IS_UNSET_POP_5, + [_GUARD_BIT_IS_UNSET_POP_5_r21] = _GUARD_BIT_IS_UNSET_POP_5, + [_GUARD_BIT_IS_UNSET_POP_5_r32] = _GUARD_BIT_IS_UNSET_POP_5, + [_GUARD_BIT_IS_UNSET_POP_6_r00] = _GUARD_BIT_IS_UNSET_POP_6, + [_GUARD_BIT_IS_UNSET_POP_6_r10] = _GUARD_BIT_IS_UNSET_POP_6, + [_GUARD_BIT_IS_UNSET_POP_6_r21] = _GUARD_BIT_IS_UNSET_POP_6, + [_GUARD_BIT_IS_UNSET_POP_6_r32] = _GUARD_BIT_IS_UNSET_POP_6, + [_GUARD_BIT_IS_UNSET_POP_7_r00] = _GUARD_BIT_IS_UNSET_POP_7, + [_GUARD_BIT_IS_UNSET_POP_7_r10] = _GUARD_BIT_IS_UNSET_POP_7, + [_GUARD_BIT_IS_UNSET_POP_7_r21] = _GUARD_BIT_IS_UNSET_POP_7, + [_GUARD_BIT_IS_UNSET_POP_7_r32] = _GUARD_BIT_IS_UNSET_POP_7, + [_GUARD_BIT_IS_UNSET_POP_r00] = _GUARD_BIT_IS_UNSET_POP, + [_GUARD_BIT_IS_UNSET_POP_r10] = _GUARD_BIT_IS_UNSET_POP, + [_GUARD_BIT_IS_UNSET_POP_r21] = _GUARD_BIT_IS_UNSET_POP, + [_GUARD_BIT_IS_UNSET_POP_r32] = _GUARD_BIT_IS_UNSET_POP, + [_GUARD_IS_NONE_POP_r00] = _GUARD_IS_NONE_POP, + [_GUARD_IS_NONE_POP_r10] = _GUARD_IS_NONE_POP, + [_GUARD_IS_NONE_POP_r21] = _GUARD_IS_NONE_POP, + [_GUARD_IS_NONE_POP_r32] = _GUARD_IS_NONE_POP, + [_GUARD_IS_NOT_NONE_POP_r10] = _GUARD_IS_NOT_NONE_POP, + [_JUMP_TO_TOP_r00] = _JUMP_TO_TOP, + [_SET_IP_r00] = _SET_IP, + [_SET_IP_r11] = _SET_IP, + [_SET_IP_r22] = _SET_IP, + [_SET_IP_r33] = _SET_IP, + [_CHECK_STACK_SPACE_OPERAND_r00] = _CHECK_STACK_SPACE_OPERAND, + [_CHECK_STACK_SPACE_OPERAND_r11] = _CHECK_STACK_SPACE_OPERAND, + [_CHECK_STACK_SPACE_OPERAND_r22] = _CHECK_STACK_SPACE_OPERAND, + [_CHECK_STACK_SPACE_OPERAND_r33] = _CHECK_STACK_SPACE_OPERAND, + [_SAVE_RETURN_OFFSET_r00] = _SAVE_RETURN_OFFSET, + [_SAVE_RETURN_OFFSET_r11] = _SAVE_RETURN_OFFSET, + [_SAVE_RETURN_OFFSET_r22] = _SAVE_RETURN_OFFSET, + [_SAVE_RETURN_OFFSET_r33] = _SAVE_RETURN_OFFSET, + [_EXIT_TRACE_r00] = _EXIT_TRACE, + [_EXIT_TRACE_r10] = _EXIT_TRACE, + [_EXIT_TRACE_r20] = _EXIT_TRACE, + [_EXIT_TRACE_r30] = _EXIT_TRACE, + [_DYNAMIC_EXIT_r00] = _DYNAMIC_EXIT, + [_DYNAMIC_EXIT_r10] = _DYNAMIC_EXIT, + [_DYNAMIC_EXIT_r20] = _DYNAMIC_EXIT, + [_DYNAMIC_EXIT_r30] = _DYNAMIC_EXIT, + [_CHECK_VALIDITY_r00] = _CHECK_VALIDITY, + [_CHECK_VALIDITY_r11] = _CHECK_VALIDITY, + [_CHECK_VALIDITY_r22] = _CHECK_VALIDITY, + [_CHECK_VALIDITY_r33] = _CHECK_VALIDITY, + [_LOAD_CONST_INLINE_r01] = _LOAD_CONST_INLINE, + [_LOAD_CONST_INLINE_r12] = _LOAD_CONST_INLINE, + [_LOAD_CONST_INLINE_r23] = _LOAD_CONST_INLINE, + [_LOAD_CONST_INLINE_BORROW_r01] = _LOAD_CONST_INLINE_BORROW, + [_LOAD_CONST_INLINE_BORROW_r12] = _LOAD_CONST_INLINE_BORROW, + [_LOAD_CONST_INLINE_BORROW_r23] = _LOAD_CONST_INLINE_BORROW, + [_RROT_3_r03] = _RROT_3, + [_RROT_3_r13] = _RROT_3, + [_RROT_3_r23] = _RROT_3, + [_RROT_3_r33] = _RROT_3, + [_START_EXECUTOR_r00] = _START_EXECUTOR, + [_MAKE_WARM_r00] = _MAKE_WARM, + [_MAKE_WARM_r11] = _MAKE_WARM, + [_MAKE_WARM_r22] = _MAKE_WARM, + [_MAKE_WARM_r33] = _MAKE_WARM, + [_FATAL_ERROR_r00] = _FATAL_ERROR, + [_FATAL_ERROR_r11] = _FATAL_ERROR, + [_FATAL_ERROR_r22] = _FATAL_ERROR, + [_FATAL_ERROR_r33] = _FATAL_ERROR, + [_DEOPT_r00] = _DEOPT, + [_DEOPT_r10] = _DEOPT, + [_DEOPT_r20] = _DEOPT, + [_DEOPT_r30] = _DEOPT, + [_HANDLE_PENDING_AND_DEOPT_r00] = _HANDLE_PENDING_AND_DEOPT, + [_HANDLE_PENDING_AND_DEOPT_r10] = _HANDLE_PENDING_AND_DEOPT, + [_HANDLE_PENDING_AND_DEOPT_r20] = _HANDLE_PENDING_AND_DEOPT, + [_HANDLE_PENDING_AND_DEOPT_r30] = _HANDLE_PENDING_AND_DEOPT, + [_ERROR_POP_N_r00] = _ERROR_POP_N, + [_SPILL_OR_RELOAD_r01] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r02] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r03] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r10] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r12] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r13] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r20] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r21] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r23] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r30] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r31] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r32] = _SPILL_OR_RELOAD, + [_TIER2_RESUME_CHECK_r00] = _TIER2_RESUME_CHECK, + [_TIER2_RESUME_CHECK_r11] = _TIER2_RESUME_CHECK, + [_TIER2_RESUME_CHECK_r22] = _TIER2_RESUME_CHECK, + [_TIER2_RESUME_CHECK_r33] = _TIER2_RESUME_CHECK, + [_COLD_EXIT_r00] = _COLD_EXIT, + [_COLD_DYNAMIC_EXIT_r00] = _COLD_DYNAMIC_EXIT, + [_GUARD_CODE_VERSION__PUSH_FRAME_r00] = _GUARD_CODE_VERSION__PUSH_FRAME, + [_GUARD_CODE_VERSION__PUSH_FRAME_r11] = _GUARD_CODE_VERSION__PUSH_FRAME, + [_GUARD_CODE_VERSION__PUSH_FRAME_r22] = _GUARD_CODE_VERSION__PUSH_FRAME, + [_GUARD_CODE_VERSION__PUSH_FRAME_r33] = _GUARD_CODE_VERSION__PUSH_FRAME, + [_GUARD_CODE_VERSION_YIELD_VALUE_r00] = _GUARD_CODE_VERSION_YIELD_VALUE, + [_GUARD_CODE_VERSION_YIELD_VALUE_r11] = _GUARD_CODE_VERSION_YIELD_VALUE, + [_GUARD_CODE_VERSION_YIELD_VALUE_r22] = _GUARD_CODE_VERSION_YIELD_VALUE, + [_GUARD_CODE_VERSION_YIELD_VALUE_r33] = _GUARD_CODE_VERSION_YIELD_VALUE, + [_GUARD_CODE_VERSION_RETURN_VALUE_r00] = _GUARD_CODE_VERSION_RETURN_VALUE, + [_GUARD_CODE_VERSION_RETURN_VALUE_r11] = _GUARD_CODE_VERSION_RETURN_VALUE, + [_GUARD_CODE_VERSION_RETURN_VALUE_r22] = _GUARD_CODE_VERSION_RETURN_VALUE, + [_GUARD_CODE_VERSION_RETURN_VALUE_r33] = _GUARD_CODE_VERSION_RETURN_VALUE, + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r00] = _GUARD_CODE_VERSION_RETURN_GENERATOR, + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r11] = _GUARD_CODE_VERSION_RETURN_GENERATOR, + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r22] = _GUARD_CODE_VERSION_RETURN_GENERATOR, + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r33] = _GUARD_CODE_VERSION_RETURN_GENERATOR, + [_GUARD_IP__PUSH_FRAME_r00] = _GUARD_IP__PUSH_FRAME, + [_GUARD_IP__PUSH_FRAME_r11] = _GUARD_IP__PUSH_FRAME, + [_GUARD_IP__PUSH_FRAME_r22] = _GUARD_IP__PUSH_FRAME, + [_GUARD_IP__PUSH_FRAME_r33] = _GUARD_IP__PUSH_FRAME, + [_GUARD_IP_YIELD_VALUE_r00] = _GUARD_IP_YIELD_VALUE, + [_GUARD_IP_YIELD_VALUE_r11] = _GUARD_IP_YIELD_VALUE, + [_GUARD_IP_YIELD_VALUE_r22] = _GUARD_IP_YIELD_VALUE, + [_GUARD_IP_YIELD_VALUE_r33] = _GUARD_IP_YIELD_VALUE, + [_GUARD_IP_RETURN_VALUE_r00] = _GUARD_IP_RETURN_VALUE, + [_GUARD_IP_RETURN_VALUE_r11] = _GUARD_IP_RETURN_VALUE, + [_GUARD_IP_RETURN_VALUE_r22] = _GUARD_IP_RETURN_VALUE, + [_GUARD_IP_RETURN_VALUE_r33] = _GUARD_IP_RETURN_VALUE, + [_GUARD_IP_RETURN_GENERATOR_r00] = _GUARD_IP_RETURN_GENERATOR, + [_GUARD_IP_RETURN_GENERATOR_r11] = _GUARD_IP_RETURN_GENERATOR, + [_GUARD_IP_RETURN_GENERATOR_r22] = _GUARD_IP_RETURN_GENERATOR, + [_GUARD_IP_RETURN_GENERATOR_r33] = _GUARD_IP_RETURN_GENERATOR, +}; + +const uint16_t _PyUop_SpillsAndReloads[4][4] = { + [0][1] = _SPILL_OR_RELOAD_r01, + [0][2] = _SPILL_OR_RELOAD_r02, + [0][3] = _SPILL_OR_RELOAD_r03, + [1][0] = _SPILL_OR_RELOAD_r10, + [1][2] = _SPILL_OR_RELOAD_r12, + [1][3] = _SPILL_OR_RELOAD_r13, + [2][0] = _SPILL_OR_RELOAD_r20, + [2][1] = _SPILL_OR_RELOAD_r21, + [2][3] = _SPILL_OR_RELOAD_r23, + [3][0] = _SPILL_OR_RELOAD_r30, + [3][1] = _SPILL_OR_RELOAD_r31, + [3][2] = _SPILL_OR_RELOAD_r32, }; -const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { +const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { + [_ALLOCATE_OBJECT] = "_ALLOCATE_OBJECT", + [_ALLOCATE_OBJECT_r00] = "_ALLOCATE_OBJECT_r00", [_BINARY_OP] = "_BINARY_OP", + [_BINARY_OP_r23] = "_BINARY_OP_r23", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", - [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", + [_BINARY_OP_ADD_FLOAT_r03] = "_BINARY_OP_ADD_FLOAT_r03", + [_BINARY_OP_ADD_FLOAT_r13] = "_BINARY_OP_ADD_FLOAT_r13", + [_BINARY_OP_ADD_FLOAT_r23] = "_BINARY_OP_ADD_FLOAT_r23", + [_BINARY_OP_ADD_FLOAT_INPLACE] = "_BINARY_OP_ADD_FLOAT_INPLACE", + [_BINARY_OP_ADD_FLOAT_INPLACE_r03] = "_BINARY_OP_ADD_FLOAT_INPLACE_r03", + [_BINARY_OP_ADD_FLOAT_INPLACE_r13] = "_BINARY_OP_ADD_FLOAT_INPLACE_r13", + [_BINARY_OP_ADD_FLOAT_INPLACE_r23] = "_BINARY_OP_ADD_FLOAT_INPLACE_r23", + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT] = "_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT", + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03] = "_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03", + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13] = "_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13", + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23] = "_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", + [_BINARY_OP_ADD_INT_r03] = "_BINARY_OP_ADD_INT_r03", + [_BINARY_OP_ADD_INT_r13] = "_BINARY_OP_ADD_INT_r13", + [_BINARY_OP_ADD_INT_r23] = "_BINARY_OP_ADD_INT_r23", + [_BINARY_OP_ADD_INT_INPLACE] = "_BINARY_OP_ADD_INT_INPLACE", + [_BINARY_OP_ADD_INT_INPLACE_r03] = "_BINARY_OP_ADD_INT_INPLACE_r03", + [_BINARY_OP_ADD_INT_INPLACE_r13] = "_BINARY_OP_ADD_INT_INPLACE_r13", + [_BINARY_OP_ADD_INT_INPLACE_r23] = "_BINARY_OP_ADD_INT_INPLACE_r23", + [_BINARY_OP_ADD_INT_INPLACE_RIGHT] = "_BINARY_OP_ADD_INT_INPLACE_RIGHT", + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r03] = "_BINARY_OP_ADD_INT_INPLACE_RIGHT_r03", + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r13] = "_BINARY_OP_ADD_INT_INPLACE_RIGHT_r13", + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r23] = "_BINARY_OP_ADD_INT_INPLACE_RIGHT_r23", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", + [_BINARY_OP_ADD_UNICODE_r03] = "_BINARY_OP_ADD_UNICODE_r03", + [_BINARY_OP_ADD_UNICODE_r13] = "_BINARY_OP_ADD_UNICODE_r13", + [_BINARY_OP_ADD_UNICODE_r23] = "_BINARY_OP_ADD_UNICODE_r23", [_BINARY_OP_EXTEND] = "_BINARY_OP_EXTEND", + [_BINARY_OP_EXTEND_r23] = "_BINARY_OP_EXTEND_r23", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", + [_BINARY_OP_INPLACE_ADD_UNICODE_r21] = "_BINARY_OP_INPLACE_ADD_UNICODE_r21", [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", - [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS", + [_BINARY_OP_MULTIPLY_FLOAT_r03] = "_BINARY_OP_MULTIPLY_FLOAT_r03", + [_BINARY_OP_MULTIPLY_FLOAT_r13] = "_BINARY_OP_MULTIPLY_FLOAT_r13", + [_BINARY_OP_MULTIPLY_FLOAT_r23] = "_BINARY_OP_MULTIPLY_FLOAT_r23", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23", [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", + [_BINARY_OP_MULTIPLY_INT_r03] = "_BINARY_OP_MULTIPLY_INT_r03", + [_BINARY_OP_MULTIPLY_INT_r13] = "_BINARY_OP_MULTIPLY_INT_r13", + [_BINARY_OP_MULTIPLY_INT_r23] = "_BINARY_OP_MULTIPLY_INT_r23", + [_BINARY_OP_MULTIPLY_INT_INPLACE] = "_BINARY_OP_MULTIPLY_INT_INPLACE", + [_BINARY_OP_MULTIPLY_INT_INPLACE_r03] = "_BINARY_OP_MULTIPLY_INT_INPLACE_r03", + [_BINARY_OP_MULTIPLY_INT_INPLACE_r13] = "_BINARY_OP_MULTIPLY_INT_INPLACE_r13", + [_BINARY_OP_MULTIPLY_INT_INPLACE_r23] = "_BINARY_OP_MULTIPLY_INT_INPLACE_r23", + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT", + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03", + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13", + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23", [_BINARY_OP_SUBSCR_CHECK_FUNC] = "_BINARY_OP_SUBSCR_CHECK_FUNC", + [_BINARY_OP_SUBSCR_CHECK_FUNC_r23] = "_BINARY_OP_SUBSCR_CHECK_FUNC_r23", [_BINARY_OP_SUBSCR_DICT] = "_BINARY_OP_SUBSCR_DICT", + [_BINARY_OP_SUBSCR_DICT_r23] = "_BINARY_OP_SUBSCR_DICT_r23", + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH] = "_BINARY_OP_SUBSCR_DICT_KNOWN_HASH", + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23] = "_BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23", [_BINARY_OP_SUBSCR_INIT_CALL] = "_BINARY_OP_SUBSCR_INIT_CALL", + [_BINARY_OP_SUBSCR_INIT_CALL_r01] = "_BINARY_OP_SUBSCR_INIT_CALL_r01", + [_BINARY_OP_SUBSCR_INIT_CALL_r11] = "_BINARY_OP_SUBSCR_INIT_CALL_r11", + [_BINARY_OP_SUBSCR_INIT_CALL_r21] = "_BINARY_OP_SUBSCR_INIT_CALL_r21", + [_BINARY_OP_SUBSCR_INIT_CALL_r31] = "_BINARY_OP_SUBSCR_INIT_CALL_r31", [_BINARY_OP_SUBSCR_LIST_INT] = "_BINARY_OP_SUBSCR_LIST_INT", + [_BINARY_OP_SUBSCR_LIST_INT_r23] = "_BINARY_OP_SUBSCR_LIST_INT_r23", [_BINARY_OP_SUBSCR_LIST_SLICE] = "_BINARY_OP_SUBSCR_LIST_SLICE", + [_BINARY_OP_SUBSCR_LIST_SLICE_r23] = "_BINARY_OP_SUBSCR_LIST_SLICE_r23", [_BINARY_OP_SUBSCR_STR_INT] = "_BINARY_OP_SUBSCR_STR_INT", + [_BINARY_OP_SUBSCR_STR_INT_r23] = "_BINARY_OP_SUBSCR_STR_INT_r23", [_BINARY_OP_SUBSCR_TUPLE_INT] = "_BINARY_OP_SUBSCR_TUPLE_INT", + [_BINARY_OP_SUBSCR_TUPLE_INT_r03] = "_BINARY_OP_SUBSCR_TUPLE_INT_r03", + [_BINARY_OP_SUBSCR_TUPLE_INT_r13] = "_BINARY_OP_SUBSCR_TUPLE_INT_r13", + [_BINARY_OP_SUBSCR_TUPLE_INT_r23] = "_BINARY_OP_SUBSCR_TUPLE_INT_r23", + [_BINARY_OP_SUBSCR_USTR_INT] = "_BINARY_OP_SUBSCR_USTR_INT", + [_BINARY_OP_SUBSCR_USTR_INT_r23] = "_BINARY_OP_SUBSCR_USTR_INT_r23", [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", - [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS", + [_BINARY_OP_SUBTRACT_FLOAT_r03] = "_BINARY_OP_SUBTRACT_FLOAT_r03", + [_BINARY_OP_SUBTRACT_FLOAT_r13] = "_BINARY_OP_SUBTRACT_FLOAT_r13", + [_BINARY_OP_SUBTRACT_FLOAT_r23] = "_BINARY_OP_SUBTRACT_FLOAT_r23", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23", [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", + [_BINARY_OP_SUBTRACT_INT_r03] = "_BINARY_OP_SUBTRACT_INT_r03", + [_BINARY_OP_SUBTRACT_INT_r13] = "_BINARY_OP_SUBTRACT_INT_r13", + [_BINARY_OP_SUBTRACT_INT_r23] = "_BINARY_OP_SUBTRACT_INT_r23", + [_BINARY_OP_SUBTRACT_INT_INPLACE] = "_BINARY_OP_SUBTRACT_INT_INPLACE", + [_BINARY_OP_SUBTRACT_INT_INPLACE_r03] = "_BINARY_OP_SUBTRACT_INT_INPLACE_r03", + [_BINARY_OP_SUBTRACT_INT_INPLACE_r13] = "_BINARY_OP_SUBTRACT_INT_INPLACE_r13", + [_BINARY_OP_SUBTRACT_INT_INPLACE_r23] = "_BINARY_OP_SUBTRACT_INT_INPLACE_r23", + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT", + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03", + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13", + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23", + [_BINARY_OP_TRUEDIV_FLOAT] = "_BINARY_OP_TRUEDIV_FLOAT", + [_BINARY_OP_TRUEDIV_FLOAT_r23] = "_BINARY_OP_TRUEDIV_FLOAT_r23", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23", [_BINARY_SLICE] = "_BINARY_SLICE", + [_BINARY_SLICE_r31] = "_BINARY_SLICE_r31", [_BUILD_INTERPOLATION] = "_BUILD_INTERPOLATION", + [_BUILD_INTERPOLATION_r01] = "_BUILD_INTERPOLATION_r01", [_BUILD_LIST] = "_BUILD_LIST", + [_BUILD_LIST_r01] = "_BUILD_LIST_r01", [_BUILD_MAP] = "_BUILD_MAP", + [_BUILD_MAP_r01] = "_BUILD_MAP_r01", [_BUILD_SET] = "_BUILD_SET", + [_BUILD_SET_r01] = "_BUILD_SET_r01", [_BUILD_SLICE] = "_BUILD_SLICE", + [_BUILD_SLICE_r01] = "_BUILD_SLICE_r01", [_BUILD_STRING] = "_BUILD_STRING", + [_BUILD_STRING_r01] = "_BUILD_STRING_r01", [_BUILD_TEMPLATE] = "_BUILD_TEMPLATE", + [_BUILD_TEMPLATE_r21] = "_BUILD_TEMPLATE_r21", [_BUILD_TUPLE] = "_BUILD_TUPLE", + [_BUILD_TUPLE_r01] = "_BUILD_TUPLE_r01", [_CALL_BUILTIN_CLASS] = "_CALL_BUILTIN_CLASS", + [_CALL_BUILTIN_CLASS_r00] = "_CALL_BUILTIN_CLASS_r00", [_CALL_BUILTIN_FAST] = "_CALL_BUILTIN_FAST", + [_CALL_BUILTIN_FAST_r00] = "_CALL_BUILTIN_FAST_r00", [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS", + [_CALL_BUILTIN_FAST_WITH_KEYWORDS_r00] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS_r00", [_CALL_BUILTIN_O] = "_CALL_BUILTIN_O", + [_CALL_BUILTIN_O_r03] = "_CALL_BUILTIN_O_r03", + [_CALL_FUNCTION_EX_NON_PY_GENERAL] = "_CALL_FUNCTION_EX_NON_PY_GENERAL", + [_CALL_FUNCTION_EX_NON_PY_GENERAL_r31] = "_CALL_FUNCTION_EX_NON_PY_GENERAL_r31", [_CALL_INTRINSIC_1] = "_CALL_INTRINSIC_1", + [_CALL_INTRINSIC_1_r12] = "_CALL_INTRINSIC_1_r12", [_CALL_INTRINSIC_2] = "_CALL_INTRINSIC_2", + [_CALL_INTRINSIC_2_r23] = "_CALL_INTRINSIC_2_r23", [_CALL_ISINSTANCE] = "_CALL_ISINSTANCE", + [_CALL_ISINSTANCE_r31] = "_CALL_ISINSTANCE_r31", [_CALL_KW_NON_PY] = "_CALL_KW_NON_PY", + [_CALL_KW_NON_PY_r11] = "_CALL_KW_NON_PY_r11", [_CALL_LEN] = "_CALL_LEN", + [_CALL_LEN_r33] = "_CALL_LEN_r33", [_CALL_LIST_APPEND] = "_CALL_LIST_APPEND", + [_CALL_LIST_APPEND_r03] = "_CALL_LIST_APPEND_r03", + [_CALL_LIST_APPEND_r13] = "_CALL_LIST_APPEND_r13", + [_CALL_LIST_APPEND_r23] = "_CALL_LIST_APPEND_r23", + [_CALL_LIST_APPEND_r33] = "_CALL_LIST_APPEND_r33", [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", + [_CALL_METHOD_DESCRIPTOR_FAST_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_r00", + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = "_CALL_METHOD_DESCRIPTOR_FAST_INLINE", + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00", [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00", [_CALL_METHOD_DESCRIPTOR_NOARGS] = "_CALL_METHOD_DESCRIPTOR_NOARGS", + [_CALL_METHOD_DESCRIPTOR_NOARGS_r03] = "_CALL_METHOD_DESCRIPTOR_NOARGS_r03", + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE] = "_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE", + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03] = "_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03", [_CALL_METHOD_DESCRIPTOR_O] = "_CALL_METHOD_DESCRIPTOR_O", + [_CALL_METHOD_DESCRIPTOR_O_r03] = "_CALL_METHOD_DESCRIPTOR_O_r03", + [_CALL_METHOD_DESCRIPTOR_O_INLINE] = "_CALL_METHOD_DESCRIPTOR_O_INLINE", + [_CALL_METHOD_DESCRIPTOR_O_INLINE_r03] = "_CALL_METHOD_DESCRIPTOR_O_INLINE_r03", [_CALL_NON_PY_GENERAL] = "_CALL_NON_PY_GENERAL", + [_CALL_NON_PY_GENERAL_r01] = "_CALL_NON_PY_GENERAL_r01", [_CALL_STR_1] = "_CALL_STR_1", + [_CALL_STR_1_r32] = "_CALL_STR_1_r32", [_CALL_TUPLE_1] = "_CALL_TUPLE_1", + [_CALL_TUPLE_1_r32] = "_CALL_TUPLE_1_r32", [_CALL_TYPE_1] = "_CALL_TYPE_1", - [_CHECK_AND_ALLOCATE_OBJECT] = "_CHECK_AND_ALLOCATE_OBJECT", + [_CALL_TYPE_1_r02] = "_CALL_TYPE_1_r02", + [_CALL_TYPE_1_r12] = "_CALL_TYPE_1_r12", + [_CALL_TYPE_1_r22] = "_CALL_TYPE_1_r22", + [_CALL_TYPE_1_r32] = "_CALL_TYPE_1_r32", [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", + [_CHECK_ATTR_CLASS_r01] = "_CHECK_ATTR_CLASS_r01", + [_CHECK_ATTR_CLASS_r11] = "_CHECK_ATTR_CLASS_r11", + [_CHECK_ATTR_CLASS_r22] = "_CHECK_ATTR_CLASS_r22", + [_CHECK_ATTR_CLASS_r33] = "_CHECK_ATTR_CLASS_r33", [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", + [_CHECK_ATTR_METHOD_LAZY_DICT_r01] = "_CHECK_ATTR_METHOD_LAZY_DICT_r01", + [_CHECK_ATTR_METHOD_LAZY_DICT_r11] = "_CHECK_ATTR_METHOD_LAZY_DICT_r11", + [_CHECK_ATTR_METHOD_LAZY_DICT_r22] = "_CHECK_ATTR_METHOD_LAZY_DICT_r22", + [_CHECK_ATTR_METHOD_LAZY_DICT_r33] = "_CHECK_ATTR_METHOD_LAZY_DICT_r33", [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00", [_CHECK_EG_MATCH] = "_CHECK_EG_MATCH", + [_CHECK_EG_MATCH_r22] = "_CHECK_EG_MATCH_r22", [_CHECK_EXC_MATCH] = "_CHECK_EXC_MATCH", - [_CHECK_FUNCTION] = "_CHECK_FUNCTION", + [_CHECK_EXC_MATCH_r22] = "_CHECK_EXC_MATCH_r22", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", + [_CHECK_FUNCTION_EXACT_ARGS_r00] = "_CHECK_FUNCTION_EXACT_ARGS_r00", [_CHECK_FUNCTION_VERSION] = "_CHECK_FUNCTION_VERSION", + [_CHECK_FUNCTION_VERSION_r00] = "_CHECK_FUNCTION_VERSION_r00", [_CHECK_FUNCTION_VERSION_INLINE] = "_CHECK_FUNCTION_VERSION_INLINE", + [_CHECK_FUNCTION_VERSION_INLINE_r00] = "_CHECK_FUNCTION_VERSION_INLINE_r00", + [_CHECK_FUNCTION_VERSION_INLINE_r11] = "_CHECK_FUNCTION_VERSION_INLINE_r11", + [_CHECK_FUNCTION_VERSION_INLINE_r22] = "_CHECK_FUNCTION_VERSION_INLINE_r22", + [_CHECK_FUNCTION_VERSION_INLINE_r33] = "_CHECK_FUNCTION_VERSION_INLINE_r33", [_CHECK_FUNCTION_VERSION_KW] = "_CHECK_FUNCTION_VERSION_KW", + [_CHECK_FUNCTION_VERSION_KW_r11] = "_CHECK_FUNCTION_VERSION_KW_r11", [_CHECK_IS_NOT_PY_CALLABLE] = "_CHECK_IS_NOT_PY_CALLABLE", + [_CHECK_IS_NOT_PY_CALLABLE_r00] = "_CHECK_IS_NOT_PY_CALLABLE_r00", + [_CHECK_IS_NOT_PY_CALLABLE_EX] = "_CHECK_IS_NOT_PY_CALLABLE_EX", + [_CHECK_IS_NOT_PY_CALLABLE_EX_r03] = "_CHECK_IS_NOT_PY_CALLABLE_EX_r03", + [_CHECK_IS_NOT_PY_CALLABLE_EX_r13] = "_CHECK_IS_NOT_PY_CALLABLE_EX_r13", + [_CHECK_IS_NOT_PY_CALLABLE_EX_r23] = "_CHECK_IS_NOT_PY_CALLABLE_EX_r23", + [_CHECK_IS_NOT_PY_CALLABLE_EX_r33] = "_CHECK_IS_NOT_PY_CALLABLE_EX_r33", [_CHECK_IS_NOT_PY_CALLABLE_KW] = "_CHECK_IS_NOT_PY_CALLABLE_KW", + [_CHECK_IS_NOT_PY_CALLABLE_KW_r11] = "_CHECK_IS_NOT_PY_CALLABLE_KW_r11", + [_CHECK_IS_PY_CALLABLE_EX] = "_CHECK_IS_PY_CALLABLE_EX", + [_CHECK_IS_PY_CALLABLE_EX_r03] = "_CHECK_IS_PY_CALLABLE_EX_r03", + [_CHECK_IS_PY_CALLABLE_EX_r13] = "_CHECK_IS_PY_CALLABLE_EX_r13", + [_CHECK_IS_PY_CALLABLE_EX_r23] = "_CHECK_IS_PY_CALLABLE_EX_r23", + [_CHECK_IS_PY_CALLABLE_EX_r33] = "_CHECK_IS_PY_CALLABLE_EX_r33", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r01] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r01", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r11] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r11", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r22] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r22", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r33] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r33", [_CHECK_METHOD_VERSION] = "_CHECK_METHOD_VERSION", + [_CHECK_METHOD_VERSION_r00] = "_CHECK_METHOD_VERSION_r00", [_CHECK_METHOD_VERSION_KW] = "_CHECK_METHOD_VERSION_KW", + [_CHECK_METHOD_VERSION_KW_r11] = "_CHECK_METHOD_VERSION_KW_r11", + [_CHECK_OBJECT] = "_CHECK_OBJECT", + [_CHECK_OBJECT_r00] = "_CHECK_OBJECT_r00", [_CHECK_PEP_523] = "_CHECK_PEP_523", + [_CHECK_PEP_523_r00] = "_CHECK_PEP_523_r00", + [_CHECK_PEP_523_r11] = "_CHECK_PEP_523_r11", + [_CHECK_PEP_523_r22] = "_CHECK_PEP_523_r22", + [_CHECK_PEP_523_r33] = "_CHECK_PEP_523_r33", [_CHECK_PERIODIC] = "_CHECK_PERIODIC", + [_CHECK_PERIODIC_r00] = "_CHECK_PERIODIC_r00", [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = "_CHECK_PERIODIC_IF_NOT_YIELD_FROM", + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00] = "_CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00", + [_CHECK_RECURSION_LIMIT] = "_CHECK_RECURSION_LIMIT", + [_CHECK_RECURSION_LIMIT_r00] = "_CHECK_RECURSION_LIMIT_r00", + [_CHECK_RECURSION_LIMIT_r11] = "_CHECK_RECURSION_LIMIT_r11", + [_CHECK_RECURSION_LIMIT_r22] = "_CHECK_RECURSION_LIMIT_r22", + [_CHECK_RECURSION_LIMIT_r33] = "_CHECK_RECURSION_LIMIT_r33", [_CHECK_RECURSION_REMAINING] = "_CHECK_RECURSION_REMAINING", + [_CHECK_RECURSION_REMAINING_r00] = "_CHECK_RECURSION_REMAINING_r00", + [_CHECK_RECURSION_REMAINING_r11] = "_CHECK_RECURSION_REMAINING_r11", + [_CHECK_RECURSION_REMAINING_r22] = "_CHECK_RECURSION_REMAINING_r22", + [_CHECK_RECURSION_REMAINING_r33] = "_CHECK_RECURSION_REMAINING_r33", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", + [_CHECK_STACK_SPACE_r00] = "_CHECK_STACK_SPACE_r00", [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND", + [_CHECK_STACK_SPACE_OPERAND_r00] = "_CHECK_STACK_SPACE_OPERAND_r00", + [_CHECK_STACK_SPACE_OPERAND_r11] = "_CHECK_STACK_SPACE_OPERAND_r11", + [_CHECK_STACK_SPACE_OPERAND_r22] = "_CHECK_STACK_SPACE_OPERAND_r22", + [_CHECK_STACK_SPACE_OPERAND_r33] = "_CHECK_STACK_SPACE_OPERAND_r33", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", + [_CHECK_VALIDITY_r00] = "_CHECK_VALIDITY_r00", + [_CHECK_VALIDITY_r11] = "_CHECK_VALIDITY_r11", + [_CHECK_VALIDITY_r22] = "_CHECK_VALIDITY_r22", + [_CHECK_VALIDITY_r33] = "_CHECK_VALIDITY_r33", + [_COLD_DYNAMIC_EXIT] = "_COLD_DYNAMIC_EXIT", + [_COLD_DYNAMIC_EXIT_r00] = "_COLD_DYNAMIC_EXIT_r00", + [_COLD_EXIT] = "_COLD_EXIT", + [_COLD_EXIT_r00] = "_COLD_EXIT_r00", [_COMPARE_OP] = "_COMPARE_OP", + [_COMPARE_OP_r21] = "_COMPARE_OP_r21", [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", + [_COMPARE_OP_FLOAT_r03] = "_COMPARE_OP_FLOAT_r03", + [_COMPARE_OP_FLOAT_r13] = "_COMPARE_OP_FLOAT_r13", + [_COMPARE_OP_FLOAT_r23] = "_COMPARE_OP_FLOAT_r23", [_COMPARE_OP_INT] = "_COMPARE_OP_INT", + [_COMPARE_OP_INT_r23] = "_COMPARE_OP_INT_r23", [_COMPARE_OP_STR] = "_COMPARE_OP_STR", + [_COMPARE_OP_STR_r23] = "_COMPARE_OP_STR_r23", [_CONTAINS_OP] = "_CONTAINS_OP", + [_CONTAINS_OP_r23] = "_CONTAINS_OP_r23", [_CONTAINS_OP_DICT] = "_CONTAINS_OP_DICT", + [_CONTAINS_OP_DICT_r23] = "_CONTAINS_OP_DICT_r23", [_CONTAINS_OP_SET] = "_CONTAINS_OP_SET", + [_CONTAINS_OP_SET_r23] = "_CONTAINS_OP_SET_r23", [_CONVERT_VALUE] = "_CONVERT_VALUE", + [_CONVERT_VALUE_r11] = "_CONVERT_VALUE_r11", [_COPY] = "_COPY", + [_COPY_r01] = "_COPY_r01", [_COPY_1] = "_COPY_1", + [_COPY_1_r02] = "_COPY_1_r02", + [_COPY_1_r12] = "_COPY_1_r12", + [_COPY_1_r23] = "_COPY_1_r23", [_COPY_2] = "_COPY_2", + [_COPY_2_r03] = "_COPY_2_r03", + [_COPY_2_r13] = "_COPY_2_r13", + [_COPY_2_r23] = "_COPY_2_r23", [_COPY_3] = "_COPY_3", + [_COPY_3_r03] = "_COPY_3_r03", + [_COPY_3_r13] = "_COPY_3_r13", + [_COPY_3_r23] = "_COPY_3_r23", + [_COPY_3_r33] = "_COPY_3_r33", [_COPY_FREE_VARS] = "_COPY_FREE_VARS", + [_COPY_FREE_VARS_r00] = "_COPY_FREE_VARS_r00", + [_COPY_FREE_VARS_r11] = "_COPY_FREE_VARS_r11", + [_COPY_FREE_VARS_r22] = "_COPY_FREE_VARS_r22", + [_COPY_FREE_VARS_r33] = "_COPY_FREE_VARS_r33", [_CREATE_INIT_FRAME] = "_CREATE_INIT_FRAME", + [_CREATE_INIT_FRAME_r01] = "_CREATE_INIT_FRAME_r01", [_DELETE_ATTR] = "_DELETE_ATTR", + [_DELETE_ATTR_r10] = "_DELETE_ATTR_r10", [_DELETE_DEREF] = "_DELETE_DEREF", + [_DELETE_DEREF_r00] = "_DELETE_DEREF_r00", [_DELETE_FAST] = "_DELETE_FAST", + [_DELETE_FAST_r00] = "_DELETE_FAST_r00", [_DELETE_GLOBAL] = "_DELETE_GLOBAL", + [_DELETE_GLOBAL_r00] = "_DELETE_GLOBAL_r00", [_DELETE_NAME] = "_DELETE_NAME", + [_DELETE_NAME_r00] = "_DELETE_NAME_r00", [_DELETE_SUBSCR] = "_DELETE_SUBSCR", + [_DELETE_SUBSCR_r20] = "_DELETE_SUBSCR_r20", [_DEOPT] = "_DEOPT", + [_DEOPT_r00] = "_DEOPT_r00", + [_DEOPT_r10] = "_DEOPT_r10", + [_DEOPT_r20] = "_DEOPT_r20", + [_DEOPT_r30] = "_DEOPT_r30", [_DICT_MERGE] = "_DICT_MERGE", + [_DICT_MERGE_r11] = "_DICT_MERGE_r11", [_DICT_UPDATE] = "_DICT_UPDATE", + [_DICT_UPDATE_r11] = "_DICT_UPDATE_r11", + [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", + [_DYNAMIC_EXIT_r00] = "_DYNAMIC_EXIT_r00", + [_DYNAMIC_EXIT_r10] = "_DYNAMIC_EXIT_r10", + [_DYNAMIC_EXIT_r20] = "_DYNAMIC_EXIT_r20", + [_DYNAMIC_EXIT_r30] = "_DYNAMIC_EXIT_r30", [_END_FOR] = "_END_FOR", + [_END_FOR_r10] = "_END_FOR_r10", [_END_SEND] = "_END_SEND", + [_END_SEND_r31] = "_END_SEND_r31", [_ERROR_POP_N] = "_ERROR_POP_N", + [_ERROR_POP_N_r00] = "_ERROR_POP_N_r00", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", + [_EXIT_INIT_CHECK_r10] = "_EXIT_INIT_CHECK_r10", [_EXIT_TRACE] = "_EXIT_TRACE", + [_EXIT_TRACE_r00] = "_EXIT_TRACE_r00", + [_EXIT_TRACE_r10] = "_EXIT_TRACE_r10", + [_EXIT_TRACE_r20] = "_EXIT_TRACE_r20", + [_EXIT_TRACE_r30] = "_EXIT_TRACE_r30", [_EXPAND_METHOD] = "_EXPAND_METHOD", + [_EXPAND_METHOD_r00] = "_EXPAND_METHOD_r00", [_EXPAND_METHOD_KW] = "_EXPAND_METHOD_KW", + [_EXPAND_METHOD_KW_r11] = "_EXPAND_METHOD_KW_r11", [_FATAL_ERROR] = "_FATAL_ERROR", + [_FATAL_ERROR_r00] = "_FATAL_ERROR_r00", + [_FATAL_ERROR_r11] = "_FATAL_ERROR_r11", + [_FATAL_ERROR_r22] = "_FATAL_ERROR_r22", + [_FATAL_ERROR_r33] = "_FATAL_ERROR_r33", [_FORMAT_SIMPLE] = "_FORMAT_SIMPLE", + [_FORMAT_SIMPLE_r11] = "_FORMAT_SIMPLE_r11", [_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC", + [_FORMAT_WITH_SPEC_r21] = "_FORMAT_WITH_SPEC_r21", [_FOR_ITER_GEN_FRAME] = "_FOR_ITER_GEN_FRAME", + [_FOR_ITER_GEN_FRAME_r03] = "_FOR_ITER_GEN_FRAME_r03", + [_FOR_ITER_GEN_FRAME_r13] = "_FOR_ITER_GEN_FRAME_r13", + [_FOR_ITER_GEN_FRAME_r23] = "_FOR_ITER_GEN_FRAME_r23", [_FOR_ITER_TIER_TWO] = "_FOR_ITER_TIER_TWO", + [_FOR_ITER_TIER_TWO_r23] = "_FOR_ITER_TIER_TWO_r23", + [_FOR_ITER_VIRTUAL_TIER_TWO] = "_FOR_ITER_VIRTUAL_TIER_TWO", + [_FOR_ITER_VIRTUAL_TIER_TWO_r23] = "_FOR_ITER_VIRTUAL_TIER_TWO_r23", [_GET_AITER] = "_GET_AITER", + [_GET_AITER_r11] = "_GET_AITER_r11", [_GET_ANEXT] = "_GET_ANEXT", + [_GET_ANEXT_r12] = "_GET_ANEXT_r12", [_GET_AWAITABLE] = "_GET_AWAITABLE", + [_GET_AWAITABLE_r11] = "_GET_AWAITABLE_r11", [_GET_ITER] = "_GET_ITER", + [_GET_ITER_r12] = "_GET_ITER_r12", + [_GET_ITER_TRAD] = "_GET_ITER_TRAD", + [_GET_ITER_TRAD_r12] = "_GET_ITER_TRAD_r12", [_GET_LEN] = "_GET_LEN", - [_GET_YIELD_FROM_ITER] = "_GET_YIELD_FROM_ITER", + [_GET_LEN_r12] = "_GET_LEN_r12", + [_GUARD_3OS_ASYNC_GEN_ASEND] = "_GUARD_3OS_ASYNC_GEN_ASEND", + [_GUARD_3OS_ASYNC_GEN_ASEND_r03] = "_GUARD_3OS_ASYNC_GEN_ASEND_r03", + [_GUARD_3OS_ASYNC_GEN_ASEND_r13] = "_GUARD_3OS_ASYNC_GEN_ASEND_r13", + [_GUARD_3OS_ASYNC_GEN_ASEND_r23] = "_GUARD_3OS_ASYNC_GEN_ASEND_r23", + [_GUARD_3OS_ASYNC_GEN_ASEND_r33] = "_GUARD_3OS_ASYNC_GEN_ASEND_r33", [_GUARD_BINARY_OP_EXTEND] = "_GUARD_BINARY_OP_EXTEND", + [_GUARD_BINARY_OP_EXTEND_r22] = "_GUARD_BINARY_OP_EXTEND_r22", + [_GUARD_BINARY_OP_EXTEND_LHS] = "_GUARD_BINARY_OP_EXTEND_LHS", + [_GUARD_BINARY_OP_EXTEND_LHS_r02] = "_GUARD_BINARY_OP_EXTEND_LHS_r02", + [_GUARD_BINARY_OP_EXTEND_LHS_r12] = "_GUARD_BINARY_OP_EXTEND_LHS_r12", + [_GUARD_BINARY_OP_EXTEND_LHS_r22] = "_GUARD_BINARY_OP_EXTEND_LHS_r22", + [_GUARD_BINARY_OP_EXTEND_LHS_r33] = "_GUARD_BINARY_OP_EXTEND_LHS_r33", + [_GUARD_BINARY_OP_EXTEND_RHS] = "_GUARD_BINARY_OP_EXTEND_RHS", + [_GUARD_BINARY_OP_EXTEND_RHS_r02] = "_GUARD_BINARY_OP_EXTEND_RHS_r02", + [_GUARD_BINARY_OP_EXTEND_RHS_r12] = "_GUARD_BINARY_OP_EXTEND_RHS_r12", + [_GUARD_BINARY_OP_EXTEND_RHS_r22] = "_GUARD_BINARY_OP_EXTEND_RHS_r22", + [_GUARD_BINARY_OP_EXTEND_RHS_r33] = "_GUARD_BINARY_OP_EXTEND_RHS_r33", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33", + [_GUARD_BIT_IS_SET_POP] = "_GUARD_BIT_IS_SET_POP", + [_GUARD_BIT_IS_SET_POP_r00] = "_GUARD_BIT_IS_SET_POP_r00", + [_GUARD_BIT_IS_SET_POP_r10] = "_GUARD_BIT_IS_SET_POP_r10", + [_GUARD_BIT_IS_SET_POP_r21] = "_GUARD_BIT_IS_SET_POP_r21", + [_GUARD_BIT_IS_SET_POP_r32] = "_GUARD_BIT_IS_SET_POP_r32", + [_GUARD_BIT_IS_SET_POP_4] = "_GUARD_BIT_IS_SET_POP_4", + [_GUARD_BIT_IS_SET_POP_4_r00] = "_GUARD_BIT_IS_SET_POP_4_r00", + [_GUARD_BIT_IS_SET_POP_4_r10] = "_GUARD_BIT_IS_SET_POP_4_r10", + [_GUARD_BIT_IS_SET_POP_4_r21] = "_GUARD_BIT_IS_SET_POP_4_r21", + [_GUARD_BIT_IS_SET_POP_4_r32] = "_GUARD_BIT_IS_SET_POP_4_r32", + [_GUARD_BIT_IS_SET_POP_5] = "_GUARD_BIT_IS_SET_POP_5", + [_GUARD_BIT_IS_SET_POP_5_r00] = "_GUARD_BIT_IS_SET_POP_5_r00", + [_GUARD_BIT_IS_SET_POP_5_r10] = "_GUARD_BIT_IS_SET_POP_5_r10", + [_GUARD_BIT_IS_SET_POP_5_r21] = "_GUARD_BIT_IS_SET_POP_5_r21", + [_GUARD_BIT_IS_SET_POP_5_r32] = "_GUARD_BIT_IS_SET_POP_5_r32", + [_GUARD_BIT_IS_SET_POP_6] = "_GUARD_BIT_IS_SET_POP_6", + [_GUARD_BIT_IS_SET_POP_6_r00] = "_GUARD_BIT_IS_SET_POP_6_r00", + [_GUARD_BIT_IS_SET_POP_6_r10] = "_GUARD_BIT_IS_SET_POP_6_r10", + [_GUARD_BIT_IS_SET_POP_6_r21] = "_GUARD_BIT_IS_SET_POP_6_r21", + [_GUARD_BIT_IS_SET_POP_6_r32] = "_GUARD_BIT_IS_SET_POP_6_r32", + [_GUARD_BIT_IS_SET_POP_7] = "_GUARD_BIT_IS_SET_POP_7", + [_GUARD_BIT_IS_SET_POP_7_r00] = "_GUARD_BIT_IS_SET_POP_7_r00", + [_GUARD_BIT_IS_SET_POP_7_r10] = "_GUARD_BIT_IS_SET_POP_7_r10", + [_GUARD_BIT_IS_SET_POP_7_r21] = "_GUARD_BIT_IS_SET_POP_7_r21", + [_GUARD_BIT_IS_SET_POP_7_r32] = "_GUARD_BIT_IS_SET_POP_7_r32", + [_GUARD_BIT_IS_UNSET_POP] = "_GUARD_BIT_IS_UNSET_POP", + [_GUARD_BIT_IS_UNSET_POP_r00] = "_GUARD_BIT_IS_UNSET_POP_r00", + [_GUARD_BIT_IS_UNSET_POP_r10] = "_GUARD_BIT_IS_UNSET_POP_r10", + [_GUARD_BIT_IS_UNSET_POP_r21] = "_GUARD_BIT_IS_UNSET_POP_r21", + [_GUARD_BIT_IS_UNSET_POP_r32] = "_GUARD_BIT_IS_UNSET_POP_r32", + [_GUARD_BIT_IS_UNSET_POP_4] = "_GUARD_BIT_IS_UNSET_POP_4", + [_GUARD_BIT_IS_UNSET_POP_4_r00] = "_GUARD_BIT_IS_UNSET_POP_4_r00", + [_GUARD_BIT_IS_UNSET_POP_4_r10] = "_GUARD_BIT_IS_UNSET_POP_4_r10", + [_GUARD_BIT_IS_UNSET_POP_4_r21] = "_GUARD_BIT_IS_UNSET_POP_4_r21", + [_GUARD_BIT_IS_UNSET_POP_4_r32] = "_GUARD_BIT_IS_UNSET_POP_4_r32", + [_GUARD_BIT_IS_UNSET_POP_5] = "_GUARD_BIT_IS_UNSET_POP_5", + [_GUARD_BIT_IS_UNSET_POP_5_r00] = "_GUARD_BIT_IS_UNSET_POP_5_r00", + [_GUARD_BIT_IS_UNSET_POP_5_r10] = "_GUARD_BIT_IS_UNSET_POP_5_r10", + [_GUARD_BIT_IS_UNSET_POP_5_r21] = "_GUARD_BIT_IS_UNSET_POP_5_r21", + [_GUARD_BIT_IS_UNSET_POP_5_r32] = "_GUARD_BIT_IS_UNSET_POP_5_r32", + [_GUARD_BIT_IS_UNSET_POP_6] = "_GUARD_BIT_IS_UNSET_POP_6", + [_GUARD_BIT_IS_UNSET_POP_6_r00] = "_GUARD_BIT_IS_UNSET_POP_6_r00", + [_GUARD_BIT_IS_UNSET_POP_6_r10] = "_GUARD_BIT_IS_UNSET_POP_6_r10", + [_GUARD_BIT_IS_UNSET_POP_6_r21] = "_GUARD_BIT_IS_UNSET_POP_6_r21", + [_GUARD_BIT_IS_UNSET_POP_6_r32] = "_GUARD_BIT_IS_UNSET_POP_6_r32", + [_GUARD_BIT_IS_UNSET_POP_7] = "_GUARD_BIT_IS_UNSET_POP_7", + [_GUARD_BIT_IS_UNSET_POP_7_r00] = "_GUARD_BIT_IS_UNSET_POP_7_r00", + [_GUARD_BIT_IS_UNSET_POP_7_r10] = "_GUARD_BIT_IS_UNSET_POP_7_r10", + [_GUARD_BIT_IS_UNSET_POP_7_r21] = "_GUARD_BIT_IS_UNSET_POP_7_r21", + [_GUARD_BIT_IS_UNSET_POP_7_r32] = "_GUARD_BIT_IS_UNSET_POP_7_r32", + [_GUARD_CALLABLE_BUILTIN_CLASS] = "_GUARD_CALLABLE_BUILTIN_CLASS", + [_GUARD_CALLABLE_BUILTIN_CLASS_r00] = "_GUARD_CALLABLE_BUILTIN_CLASS_r00", + [_GUARD_CALLABLE_BUILTIN_FAST] = "_GUARD_CALLABLE_BUILTIN_FAST", + [_GUARD_CALLABLE_BUILTIN_FAST_r00] = "_GUARD_CALLABLE_BUILTIN_FAST_r00", + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS] = "_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS", + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00] = "_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00", + [_GUARD_CALLABLE_BUILTIN_O] = "_GUARD_CALLABLE_BUILTIN_O", + [_GUARD_CALLABLE_BUILTIN_O_r00] = "_GUARD_CALLABLE_BUILTIN_O_r00", [_GUARD_CALLABLE_ISINSTANCE] = "_GUARD_CALLABLE_ISINSTANCE", + [_GUARD_CALLABLE_ISINSTANCE_r03] = "_GUARD_CALLABLE_ISINSTANCE_r03", + [_GUARD_CALLABLE_ISINSTANCE_r13] = "_GUARD_CALLABLE_ISINSTANCE_r13", + [_GUARD_CALLABLE_ISINSTANCE_r23] = "_GUARD_CALLABLE_ISINSTANCE_r23", + [_GUARD_CALLABLE_ISINSTANCE_r33] = "_GUARD_CALLABLE_ISINSTANCE_r33", [_GUARD_CALLABLE_LEN] = "_GUARD_CALLABLE_LEN", + [_GUARD_CALLABLE_LEN_r03] = "_GUARD_CALLABLE_LEN_r03", + [_GUARD_CALLABLE_LEN_r13] = "_GUARD_CALLABLE_LEN_r13", + [_GUARD_CALLABLE_LEN_r23] = "_GUARD_CALLABLE_LEN_r23", + [_GUARD_CALLABLE_LEN_r33] = "_GUARD_CALLABLE_LEN_r33", [_GUARD_CALLABLE_LIST_APPEND] = "_GUARD_CALLABLE_LIST_APPEND", + [_GUARD_CALLABLE_LIST_APPEND_r03] = "_GUARD_CALLABLE_LIST_APPEND_r03", + [_GUARD_CALLABLE_LIST_APPEND_r13] = "_GUARD_CALLABLE_LIST_APPEND_r13", + [_GUARD_CALLABLE_LIST_APPEND_r23] = "_GUARD_CALLABLE_LIST_APPEND_r23", + [_GUARD_CALLABLE_LIST_APPEND_r33] = "_GUARD_CALLABLE_LIST_APPEND_r33", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_O", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00", [_GUARD_CALLABLE_STR_1] = "_GUARD_CALLABLE_STR_1", + [_GUARD_CALLABLE_STR_1_r03] = "_GUARD_CALLABLE_STR_1_r03", + [_GUARD_CALLABLE_STR_1_r13] = "_GUARD_CALLABLE_STR_1_r13", + [_GUARD_CALLABLE_STR_1_r23] = "_GUARD_CALLABLE_STR_1_r23", + [_GUARD_CALLABLE_STR_1_r33] = "_GUARD_CALLABLE_STR_1_r33", [_GUARD_CALLABLE_TUPLE_1] = "_GUARD_CALLABLE_TUPLE_1", + [_GUARD_CALLABLE_TUPLE_1_r03] = "_GUARD_CALLABLE_TUPLE_1_r03", + [_GUARD_CALLABLE_TUPLE_1_r13] = "_GUARD_CALLABLE_TUPLE_1_r13", + [_GUARD_CALLABLE_TUPLE_1_r23] = "_GUARD_CALLABLE_TUPLE_1_r23", + [_GUARD_CALLABLE_TUPLE_1_r33] = "_GUARD_CALLABLE_TUPLE_1_r33", [_GUARD_CALLABLE_TYPE_1] = "_GUARD_CALLABLE_TYPE_1", + [_GUARD_CALLABLE_TYPE_1_r03] = "_GUARD_CALLABLE_TYPE_1_r03", + [_GUARD_CALLABLE_TYPE_1_r13] = "_GUARD_CALLABLE_TYPE_1_r13", + [_GUARD_CALLABLE_TYPE_1_r23] = "_GUARD_CALLABLE_TYPE_1_r23", + [_GUARD_CALLABLE_TYPE_1_r33] = "_GUARD_CALLABLE_TYPE_1_r33", + [_GUARD_CODE_VERSION_RETURN_GENERATOR] = "_GUARD_CODE_VERSION_RETURN_GENERATOR", + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r00] = "_GUARD_CODE_VERSION_RETURN_GENERATOR_r00", + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r11] = "_GUARD_CODE_VERSION_RETURN_GENERATOR_r11", + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r22] = "_GUARD_CODE_VERSION_RETURN_GENERATOR_r22", + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r33] = "_GUARD_CODE_VERSION_RETURN_GENERATOR_r33", + [_GUARD_CODE_VERSION_RETURN_VALUE] = "_GUARD_CODE_VERSION_RETURN_VALUE", + [_GUARD_CODE_VERSION_RETURN_VALUE_r00] = "_GUARD_CODE_VERSION_RETURN_VALUE_r00", + [_GUARD_CODE_VERSION_RETURN_VALUE_r11] = "_GUARD_CODE_VERSION_RETURN_VALUE_r11", + [_GUARD_CODE_VERSION_RETURN_VALUE_r22] = "_GUARD_CODE_VERSION_RETURN_VALUE_r22", + [_GUARD_CODE_VERSION_RETURN_VALUE_r33] = "_GUARD_CODE_VERSION_RETURN_VALUE_r33", + [_GUARD_CODE_VERSION_YIELD_VALUE] = "_GUARD_CODE_VERSION_YIELD_VALUE", + [_GUARD_CODE_VERSION_YIELD_VALUE_r00] = "_GUARD_CODE_VERSION_YIELD_VALUE_r00", + [_GUARD_CODE_VERSION_YIELD_VALUE_r11] = "_GUARD_CODE_VERSION_YIELD_VALUE_r11", + [_GUARD_CODE_VERSION_YIELD_VALUE_r22] = "_GUARD_CODE_VERSION_YIELD_VALUE_r22", + [_GUARD_CODE_VERSION_YIELD_VALUE_r33] = "_GUARD_CODE_VERSION_YIELD_VALUE_r33", + [_GUARD_CODE_VERSION__PUSH_FRAME] = "_GUARD_CODE_VERSION__PUSH_FRAME", + [_GUARD_CODE_VERSION__PUSH_FRAME_r00] = "_GUARD_CODE_VERSION__PUSH_FRAME_r00", + [_GUARD_CODE_VERSION__PUSH_FRAME_r11] = "_GUARD_CODE_VERSION__PUSH_FRAME_r11", + [_GUARD_CODE_VERSION__PUSH_FRAME_r22] = "_GUARD_CODE_VERSION__PUSH_FRAME_r22", + [_GUARD_CODE_VERSION__PUSH_FRAME_r33] = "_GUARD_CODE_VERSION__PUSH_FRAME_r33", [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", + [_GUARD_DORV_NO_DICT_r01] = "_GUARD_DORV_NO_DICT_r01", + [_GUARD_DORV_NO_DICT_r11] = "_GUARD_DORV_NO_DICT_r11", + [_GUARD_DORV_NO_DICT_r22] = "_GUARD_DORV_NO_DICT_r22", + [_GUARD_DORV_NO_DICT_r33] = "_GUARD_DORV_NO_DICT_r33", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_GLOBALS_VERSION_r00] = "_GUARD_GLOBALS_VERSION_r00", + [_GUARD_GLOBALS_VERSION_r11] = "_GUARD_GLOBALS_VERSION_r11", + [_GUARD_GLOBALS_VERSION_r22] = "_GUARD_GLOBALS_VERSION_r22", + [_GUARD_GLOBALS_VERSION_r33] = "_GUARD_GLOBALS_VERSION_r33", + [_GUARD_IP_RETURN_GENERATOR] = "_GUARD_IP_RETURN_GENERATOR", + [_GUARD_IP_RETURN_GENERATOR_r00] = "_GUARD_IP_RETURN_GENERATOR_r00", + [_GUARD_IP_RETURN_GENERATOR_r11] = "_GUARD_IP_RETURN_GENERATOR_r11", + [_GUARD_IP_RETURN_GENERATOR_r22] = "_GUARD_IP_RETURN_GENERATOR_r22", + [_GUARD_IP_RETURN_GENERATOR_r33] = "_GUARD_IP_RETURN_GENERATOR_r33", + [_GUARD_IP_RETURN_VALUE] = "_GUARD_IP_RETURN_VALUE", + [_GUARD_IP_RETURN_VALUE_r00] = "_GUARD_IP_RETURN_VALUE_r00", + [_GUARD_IP_RETURN_VALUE_r11] = "_GUARD_IP_RETURN_VALUE_r11", + [_GUARD_IP_RETURN_VALUE_r22] = "_GUARD_IP_RETURN_VALUE_r22", + [_GUARD_IP_RETURN_VALUE_r33] = "_GUARD_IP_RETURN_VALUE_r33", + [_GUARD_IP_YIELD_VALUE] = "_GUARD_IP_YIELD_VALUE", + [_GUARD_IP_YIELD_VALUE_r00] = "_GUARD_IP_YIELD_VALUE_r00", + [_GUARD_IP_YIELD_VALUE_r11] = "_GUARD_IP_YIELD_VALUE_r11", + [_GUARD_IP_YIELD_VALUE_r22] = "_GUARD_IP_YIELD_VALUE_r22", + [_GUARD_IP_YIELD_VALUE_r33] = "_GUARD_IP_YIELD_VALUE_r33", + [_GUARD_IP__PUSH_FRAME] = "_GUARD_IP__PUSH_FRAME", + [_GUARD_IP__PUSH_FRAME_r00] = "_GUARD_IP__PUSH_FRAME_r00", + [_GUARD_IP__PUSH_FRAME_r11] = "_GUARD_IP__PUSH_FRAME_r11", + [_GUARD_IP__PUSH_FRAME_r22] = "_GUARD_IP__PUSH_FRAME_r22", + [_GUARD_IP__PUSH_FRAME_r33] = "_GUARD_IP__PUSH_FRAME_r33", [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", + [_GUARD_IS_FALSE_POP_r00] = "_GUARD_IS_FALSE_POP_r00", + [_GUARD_IS_FALSE_POP_r10] = "_GUARD_IS_FALSE_POP_r10", + [_GUARD_IS_FALSE_POP_r21] = "_GUARD_IS_FALSE_POP_r21", + [_GUARD_IS_FALSE_POP_r32] = "_GUARD_IS_FALSE_POP_r32", [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", + [_GUARD_IS_NONE_POP_r00] = "_GUARD_IS_NONE_POP_r00", + [_GUARD_IS_NONE_POP_r10] = "_GUARD_IS_NONE_POP_r10", + [_GUARD_IS_NONE_POP_r21] = "_GUARD_IS_NONE_POP_r21", + [_GUARD_IS_NONE_POP_r32] = "_GUARD_IS_NONE_POP_r32", [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", + [_GUARD_IS_NOT_NONE_POP_r10] = "_GUARD_IS_NOT_NONE_POP_r10", [_GUARD_IS_TRUE_POP] = "_GUARD_IS_TRUE_POP", - [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", - [_GUARD_NOS_DICT] = "_GUARD_NOS_DICT", + [_GUARD_IS_TRUE_POP_r00] = "_GUARD_IS_TRUE_POP_r00", + [_GUARD_IS_TRUE_POP_r10] = "_GUARD_IS_TRUE_POP_r10", + [_GUARD_IS_TRUE_POP_r21] = "_GUARD_IS_TRUE_POP_r21", + [_GUARD_IS_TRUE_POP_r32] = "_GUARD_IS_TRUE_POP_r32", + [_GUARD_ITERATOR] = "_GUARD_ITERATOR", + [_GUARD_ITERATOR_r01] = "_GUARD_ITERATOR_r01", + [_GUARD_ITERATOR_r11] = "_GUARD_ITERATOR_r11", + [_GUARD_ITERATOR_r22] = "_GUARD_ITERATOR_r22", + [_GUARD_ITERATOR_r33] = "_GUARD_ITERATOR_r33", + [_GUARD_ITER_VIRTUAL] = "_GUARD_ITER_VIRTUAL", + [_GUARD_ITER_VIRTUAL_r01] = "_GUARD_ITER_VIRTUAL_r01", + [_GUARD_ITER_VIRTUAL_r11] = "_GUARD_ITER_VIRTUAL_r11", + [_GUARD_ITER_VIRTUAL_r22] = "_GUARD_ITER_VIRTUAL_r22", + [_GUARD_ITER_VIRTUAL_r33] = "_GUARD_ITER_VIRTUAL_r33", + [_GUARD_LOAD_SUPER_ATTR_METHOD] = "_GUARD_LOAD_SUPER_ATTR_METHOD", + [_GUARD_LOAD_SUPER_ATTR_METHOD_r03] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r03", + [_GUARD_LOAD_SUPER_ATTR_METHOD_r13] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r13", + [_GUARD_LOAD_SUPER_ATTR_METHOD_r23] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r23", + [_GUARD_LOAD_SUPER_ATTR_METHOD_r33] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r33", + [_GUARD_NOS_COMPACT_ASCII] = "_GUARD_NOS_COMPACT_ASCII", + [_GUARD_NOS_COMPACT_ASCII_r02] = "_GUARD_NOS_COMPACT_ASCII_r02", + [_GUARD_NOS_COMPACT_ASCII_r12] = "_GUARD_NOS_COMPACT_ASCII_r12", + [_GUARD_NOS_COMPACT_ASCII_r22] = "_GUARD_NOS_COMPACT_ASCII_r22", + [_GUARD_NOS_COMPACT_ASCII_r33] = "_GUARD_NOS_COMPACT_ASCII_r33", + [_GUARD_NOS_DICT_STORE_SUBSCRIPT] = "_GUARD_NOS_DICT_STORE_SUBSCRIPT", + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r03] = "_GUARD_NOS_DICT_STORE_SUBSCRIPT_r03", + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r13] = "_GUARD_NOS_DICT_STORE_SUBSCRIPT_r13", + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r23] = "_GUARD_NOS_DICT_STORE_SUBSCRIPT_r23", + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r33] = "_GUARD_NOS_DICT_STORE_SUBSCRIPT_r33", + [_GUARD_NOS_DICT_SUBSCRIPT] = "_GUARD_NOS_DICT_SUBSCRIPT", + [_GUARD_NOS_DICT_SUBSCRIPT_r02] = "_GUARD_NOS_DICT_SUBSCRIPT_r02", + [_GUARD_NOS_DICT_SUBSCRIPT_r12] = "_GUARD_NOS_DICT_SUBSCRIPT_r12", + [_GUARD_NOS_DICT_SUBSCRIPT_r22] = "_GUARD_NOS_DICT_SUBSCRIPT_r22", + [_GUARD_NOS_DICT_SUBSCRIPT_r33] = "_GUARD_NOS_DICT_SUBSCRIPT_r33", [_GUARD_NOS_FLOAT] = "_GUARD_NOS_FLOAT", + [_GUARD_NOS_FLOAT_r02] = "_GUARD_NOS_FLOAT_r02", + [_GUARD_NOS_FLOAT_r12] = "_GUARD_NOS_FLOAT_r12", + [_GUARD_NOS_FLOAT_r22] = "_GUARD_NOS_FLOAT_r22", + [_GUARD_NOS_FLOAT_r33] = "_GUARD_NOS_FLOAT_r33", [_GUARD_NOS_INT] = "_GUARD_NOS_INT", + [_GUARD_NOS_INT_r02] = "_GUARD_NOS_INT_r02", + [_GUARD_NOS_INT_r12] = "_GUARD_NOS_INT_r12", + [_GUARD_NOS_INT_r22] = "_GUARD_NOS_INT_r22", + [_GUARD_NOS_INT_r33] = "_GUARD_NOS_INT_r33", + [_GUARD_NOS_ITER_VIRTUAL] = "_GUARD_NOS_ITER_VIRTUAL", + [_GUARD_NOS_ITER_VIRTUAL_r02] = "_GUARD_NOS_ITER_VIRTUAL_r02", + [_GUARD_NOS_ITER_VIRTUAL_r12] = "_GUARD_NOS_ITER_VIRTUAL_r12", + [_GUARD_NOS_ITER_VIRTUAL_r22] = "_GUARD_NOS_ITER_VIRTUAL_r22", + [_GUARD_NOS_ITER_VIRTUAL_r33] = "_GUARD_NOS_ITER_VIRTUAL_r33", [_GUARD_NOS_LIST] = "_GUARD_NOS_LIST", + [_GUARD_NOS_LIST_r02] = "_GUARD_NOS_LIST_r02", + [_GUARD_NOS_LIST_r12] = "_GUARD_NOS_LIST_r12", + [_GUARD_NOS_LIST_r22] = "_GUARD_NOS_LIST_r22", + [_GUARD_NOS_LIST_r33] = "_GUARD_NOS_LIST_r33", [_GUARD_NOS_NOT_NULL] = "_GUARD_NOS_NOT_NULL", + [_GUARD_NOS_NOT_NULL_r02] = "_GUARD_NOS_NOT_NULL_r02", + [_GUARD_NOS_NOT_NULL_r12] = "_GUARD_NOS_NOT_NULL_r12", + [_GUARD_NOS_NOT_NULL_r22] = "_GUARD_NOS_NOT_NULL_r22", + [_GUARD_NOS_NOT_NULL_r33] = "_GUARD_NOS_NOT_NULL_r33", [_GUARD_NOS_NULL] = "_GUARD_NOS_NULL", + [_GUARD_NOS_NULL_r02] = "_GUARD_NOS_NULL_r02", + [_GUARD_NOS_NULL_r12] = "_GUARD_NOS_NULL_r12", + [_GUARD_NOS_NULL_r22] = "_GUARD_NOS_NULL_r22", + [_GUARD_NOS_NULL_r33] = "_GUARD_NOS_NULL_r33", [_GUARD_NOS_OVERFLOWED] = "_GUARD_NOS_OVERFLOWED", + [_GUARD_NOS_OVERFLOWED_r02] = "_GUARD_NOS_OVERFLOWED_r02", + [_GUARD_NOS_OVERFLOWED_r12] = "_GUARD_NOS_OVERFLOWED_r12", + [_GUARD_NOS_OVERFLOWED_r22] = "_GUARD_NOS_OVERFLOWED_r22", + [_GUARD_NOS_OVERFLOWED_r33] = "_GUARD_NOS_OVERFLOWED_r33", [_GUARD_NOS_TUPLE] = "_GUARD_NOS_TUPLE", + [_GUARD_NOS_TUPLE_r02] = "_GUARD_NOS_TUPLE_r02", + [_GUARD_NOS_TUPLE_r12] = "_GUARD_NOS_TUPLE_r12", + [_GUARD_NOS_TUPLE_r22] = "_GUARD_NOS_TUPLE_r22", + [_GUARD_NOS_TUPLE_r33] = "_GUARD_NOS_TUPLE_r33", + [_GUARD_NOS_TYPE_VERSION] = "_GUARD_NOS_TYPE_VERSION", + [_GUARD_NOS_TYPE_VERSION_r02] = "_GUARD_NOS_TYPE_VERSION_r02", + [_GUARD_NOS_TYPE_VERSION_r12] = "_GUARD_NOS_TYPE_VERSION_r12", + [_GUARD_NOS_TYPE_VERSION_r22] = "_GUARD_NOS_TYPE_VERSION_r22", + [_GUARD_NOS_TYPE_VERSION_r33] = "_GUARD_NOS_TYPE_VERSION_r33", [_GUARD_NOS_UNICODE] = "_GUARD_NOS_UNICODE", + [_GUARD_NOS_UNICODE_r02] = "_GUARD_NOS_UNICODE_r02", + [_GUARD_NOS_UNICODE_r12] = "_GUARD_NOS_UNICODE_r12", + [_GUARD_NOS_UNICODE_r22] = "_GUARD_NOS_UNICODE_r22", + [_GUARD_NOS_UNICODE_r33] = "_GUARD_NOS_UNICODE_r33", [_GUARD_NOT_EXHAUSTED_LIST] = "_GUARD_NOT_EXHAUSTED_LIST", + [_GUARD_NOT_EXHAUSTED_LIST_r02] = "_GUARD_NOT_EXHAUSTED_LIST_r02", + [_GUARD_NOT_EXHAUSTED_LIST_r12] = "_GUARD_NOT_EXHAUSTED_LIST_r12", + [_GUARD_NOT_EXHAUSTED_LIST_r22] = "_GUARD_NOT_EXHAUSTED_LIST_r22", + [_GUARD_NOT_EXHAUSTED_LIST_r33] = "_GUARD_NOT_EXHAUSTED_LIST_r33", [_GUARD_NOT_EXHAUSTED_RANGE] = "_GUARD_NOT_EXHAUSTED_RANGE", + [_GUARD_NOT_EXHAUSTED_RANGE_r02] = "_GUARD_NOT_EXHAUSTED_RANGE_r02", + [_GUARD_NOT_EXHAUSTED_RANGE_r12] = "_GUARD_NOT_EXHAUSTED_RANGE_r12", + [_GUARD_NOT_EXHAUSTED_RANGE_r22] = "_GUARD_NOT_EXHAUSTED_RANGE_r22", + [_GUARD_NOT_EXHAUSTED_RANGE_r33] = "_GUARD_NOT_EXHAUSTED_RANGE_r33", [_GUARD_NOT_EXHAUSTED_TUPLE] = "_GUARD_NOT_EXHAUSTED_TUPLE", + [_GUARD_NOT_EXHAUSTED_TUPLE_r02] = "_GUARD_NOT_EXHAUSTED_TUPLE_r02", + [_GUARD_NOT_EXHAUSTED_TUPLE_r12] = "_GUARD_NOT_EXHAUSTED_TUPLE_r12", + [_GUARD_NOT_EXHAUSTED_TUPLE_r22] = "_GUARD_NOT_EXHAUSTED_TUPLE_r22", + [_GUARD_NOT_EXHAUSTED_TUPLE_r33] = "_GUARD_NOT_EXHAUSTED_TUPLE_r33", [_GUARD_THIRD_NULL] = "_GUARD_THIRD_NULL", + [_GUARD_THIRD_NULL_r03] = "_GUARD_THIRD_NULL_r03", + [_GUARD_THIRD_NULL_r13] = "_GUARD_THIRD_NULL_r13", + [_GUARD_THIRD_NULL_r23] = "_GUARD_THIRD_NULL_r23", + [_GUARD_THIRD_NULL_r33] = "_GUARD_THIRD_NULL_r33", + [_GUARD_TOS_ANY_DICT] = "_GUARD_TOS_ANY_DICT", + [_GUARD_TOS_ANY_DICT_r01] = "_GUARD_TOS_ANY_DICT_r01", + [_GUARD_TOS_ANY_DICT_r11] = "_GUARD_TOS_ANY_DICT_r11", + [_GUARD_TOS_ANY_DICT_r22] = "_GUARD_TOS_ANY_DICT_r22", + [_GUARD_TOS_ANY_DICT_r33] = "_GUARD_TOS_ANY_DICT_r33", [_GUARD_TOS_ANY_SET] = "_GUARD_TOS_ANY_SET", + [_GUARD_TOS_ANY_SET_r01] = "_GUARD_TOS_ANY_SET_r01", + [_GUARD_TOS_ANY_SET_r11] = "_GUARD_TOS_ANY_SET_r11", + [_GUARD_TOS_ANY_SET_r22] = "_GUARD_TOS_ANY_SET_r22", + [_GUARD_TOS_ANY_SET_r33] = "_GUARD_TOS_ANY_SET_r33", [_GUARD_TOS_DICT] = "_GUARD_TOS_DICT", + [_GUARD_TOS_DICT_r01] = "_GUARD_TOS_DICT_r01", + [_GUARD_TOS_DICT_r11] = "_GUARD_TOS_DICT_r11", + [_GUARD_TOS_DICT_r22] = "_GUARD_TOS_DICT_r22", + [_GUARD_TOS_DICT_r33] = "_GUARD_TOS_DICT_r33", [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", + [_GUARD_TOS_FLOAT_r01] = "_GUARD_TOS_FLOAT_r01", + [_GUARD_TOS_FLOAT_r11] = "_GUARD_TOS_FLOAT_r11", + [_GUARD_TOS_FLOAT_r22] = "_GUARD_TOS_FLOAT_r22", + [_GUARD_TOS_FLOAT_r33] = "_GUARD_TOS_FLOAT_r33", + [_GUARD_TOS_FROZENDICT] = "_GUARD_TOS_FROZENDICT", + [_GUARD_TOS_FROZENDICT_r01] = "_GUARD_TOS_FROZENDICT_r01", + [_GUARD_TOS_FROZENDICT_r11] = "_GUARD_TOS_FROZENDICT_r11", + [_GUARD_TOS_FROZENDICT_r22] = "_GUARD_TOS_FROZENDICT_r22", + [_GUARD_TOS_FROZENDICT_r33] = "_GUARD_TOS_FROZENDICT_r33", + [_GUARD_TOS_FROZENSET] = "_GUARD_TOS_FROZENSET", + [_GUARD_TOS_FROZENSET_r01] = "_GUARD_TOS_FROZENSET_r01", + [_GUARD_TOS_FROZENSET_r11] = "_GUARD_TOS_FROZENSET_r11", + [_GUARD_TOS_FROZENSET_r22] = "_GUARD_TOS_FROZENSET_r22", + [_GUARD_TOS_FROZENSET_r33] = "_GUARD_TOS_FROZENSET_r33", [_GUARD_TOS_INT] = "_GUARD_TOS_INT", + [_GUARD_TOS_INT_r01] = "_GUARD_TOS_INT_r01", + [_GUARD_TOS_INT_r11] = "_GUARD_TOS_INT_r11", + [_GUARD_TOS_INT_r22] = "_GUARD_TOS_INT_r22", + [_GUARD_TOS_INT_r33] = "_GUARD_TOS_INT_r33", + [_GUARD_TOS_IS_NONE] = "_GUARD_TOS_IS_NONE", + [_GUARD_TOS_IS_NONE_r01] = "_GUARD_TOS_IS_NONE_r01", + [_GUARD_TOS_IS_NONE_r11] = "_GUARD_TOS_IS_NONE_r11", + [_GUARD_TOS_IS_NONE_r22] = "_GUARD_TOS_IS_NONE_r22", + [_GUARD_TOS_IS_NONE_r33] = "_GUARD_TOS_IS_NONE_r33", [_GUARD_TOS_LIST] = "_GUARD_TOS_LIST", + [_GUARD_TOS_LIST_r01] = "_GUARD_TOS_LIST_r01", + [_GUARD_TOS_LIST_r11] = "_GUARD_TOS_LIST_r11", + [_GUARD_TOS_LIST_r22] = "_GUARD_TOS_LIST_r22", + [_GUARD_TOS_LIST_r33] = "_GUARD_TOS_LIST_r33", + [_GUARD_TOS_NOT_NULL] = "_GUARD_TOS_NOT_NULL", + [_GUARD_TOS_NOT_NULL_r01] = "_GUARD_TOS_NOT_NULL_r01", + [_GUARD_TOS_NOT_NULL_r11] = "_GUARD_TOS_NOT_NULL_r11", + [_GUARD_TOS_NOT_NULL_r22] = "_GUARD_TOS_NOT_NULL_r22", + [_GUARD_TOS_NOT_NULL_r33] = "_GUARD_TOS_NOT_NULL_r33", [_GUARD_TOS_OVERFLOWED] = "_GUARD_TOS_OVERFLOWED", + [_GUARD_TOS_OVERFLOWED_r01] = "_GUARD_TOS_OVERFLOWED_r01", + [_GUARD_TOS_OVERFLOWED_r11] = "_GUARD_TOS_OVERFLOWED_r11", + [_GUARD_TOS_OVERFLOWED_r22] = "_GUARD_TOS_OVERFLOWED_r22", + [_GUARD_TOS_OVERFLOWED_r33] = "_GUARD_TOS_OVERFLOWED_r33", + [_GUARD_TOS_SET] = "_GUARD_TOS_SET", + [_GUARD_TOS_SET_r01] = "_GUARD_TOS_SET_r01", + [_GUARD_TOS_SET_r11] = "_GUARD_TOS_SET_r11", + [_GUARD_TOS_SET_r22] = "_GUARD_TOS_SET_r22", + [_GUARD_TOS_SET_r33] = "_GUARD_TOS_SET_r33", [_GUARD_TOS_SLICE] = "_GUARD_TOS_SLICE", + [_GUARD_TOS_SLICE_r01] = "_GUARD_TOS_SLICE_r01", + [_GUARD_TOS_SLICE_r11] = "_GUARD_TOS_SLICE_r11", + [_GUARD_TOS_SLICE_r22] = "_GUARD_TOS_SLICE_r22", + [_GUARD_TOS_SLICE_r33] = "_GUARD_TOS_SLICE_r33", [_GUARD_TOS_TUPLE] = "_GUARD_TOS_TUPLE", + [_GUARD_TOS_TUPLE_r01] = "_GUARD_TOS_TUPLE_r01", + [_GUARD_TOS_TUPLE_r11] = "_GUARD_TOS_TUPLE_r11", + [_GUARD_TOS_TUPLE_r22] = "_GUARD_TOS_TUPLE_r22", + [_GUARD_TOS_TUPLE_r33] = "_GUARD_TOS_TUPLE_r33", [_GUARD_TOS_UNICODE] = "_GUARD_TOS_UNICODE", + [_GUARD_TOS_UNICODE_r01] = "_GUARD_TOS_UNICODE_r01", + [_GUARD_TOS_UNICODE_r11] = "_GUARD_TOS_UNICODE_r11", + [_GUARD_TOS_UNICODE_r22] = "_GUARD_TOS_UNICODE_r22", + [_GUARD_TOS_UNICODE_r33] = "_GUARD_TOS_UNICODE_r33", + [_GUARD_TYPE] = "_GUARD_TYPE", + [_GUARD_TYPE_r01] = "_GUARD_TYPE_r01", + [_GUARD_TYPE_r11] = "_GUARD_TYPE_r11", + [_GUARD_TYPE_r22] = "_GUARD_TYPE_r22", + [_GUARD_TYPE_r33] = "_GUARD_TYPE_r33", + [_GUARD_TYPE_ITER] = "_GUARD_TYPE_ITER", + [_GUARD_TYPE_ITER_r02] = "_GUARD_TYPE_ITER_r02", + [_GUARD_TYPE_ITER_r12] = "_GUARD_TYPE_ITER_r12", + [_GUARD_TYPE_ITER_r22] = "_GUARD_TYPE_ITER_r22", + [_GUARD_TYPE_ITER_r33] = "_GUARD_TYPE_ITER_r33", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", - [_GUARD_TYPE_VERSION_AND_LOCK] = "_GUARD_TYPE_VERSION_AND_LOCK", + [_GUARD_TYPE_VERSION_r01] = "_GUARD_TYPE_VERSION_r01", + [_GUARD_TYPE_VERSION_r11] = "_GUARD_TYPE_VERSION_r11", + [_GUARD_TYPE_VERSION_r22] = "_GUARD_TYPE_VERSION_r22", + [_GUARD_TYPE_VERSION_r33] = "_GUARD_TYPE_VERSION_r33", + [_GUARD_TYPE_VERSION_LOCKED] = "_GUARD_TYPE_VERSION_LOCKED", + [_GUARD_TYPE_VERSION_LOCKED_r01] = "_GUARD_TYPE_VERSION_LOCKED_r01", + [_GUARD_TYPE_VERSION_LOCKED_r11] = "_GUARD_TYPE_VERSION_LOCKED_r11", + [_GUARD_TYPE_VERSION_LOCKED_r22] = "_GUARD_TYPE_VERSION_LOCKED_r22", + [_GUARD_TYPE_VERSION_LOCKED_r33] = "_GUARD_TYPE_VERSION_LOCKED_r33", + [_HANDLE_PENDING_AND_DEOPT] = "_HANDLE_PENDING_AND_DEOPT", + [_HANDLE_PENDING_AND_DEOPT_r00] = "_HANDLE_PENDING_AND_DEOPT_r00", + [_HANDLE_PENDING_AND_DEOPT_r10] = "_HANDLE_PENDING_AND_DEOPT_r10", + [_HANDLE_PENDING_AND_DEOPT_r20] = "_HANDLE_PENDING_AND_DEOPT_r20", + [_HANDLE_PENDING_AND_DEOPT_r30] = "_HANDLE_PENDING_AND_DEOPT_r30", [_IMPORT_FROM] = "_IMPORT_FROM", + [_IMPORT_FROM_r12] = "_IMPORT_FROM_r12", [_IMPORT_NAME] = "_IMPORT_NAME", + [_IMPORT_NAME_r21] = "_IMPORT_NAME_r21", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00", [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", + [_INIT_CALL_PY_EXACT_ARGS_r01] = "_INIT_CALL_PY_EXACT_ARGS_r01", [_INIT_CALL_PY_EXACT_ARGS_0] = "_INIT_CALL_PY_EXACT_ARGS_0", + [_INIT_CALL_PY_EXACT_ARGS_0_r01] = "_INIT_CALL_PY_EXACT_ARGS_0_r01", [_INIT_CALL_PY_EXACT_ARGS_1] = "_INIT_CALL_PY_EXACT_ARGS_1", + [_INIT_CALL_PY_EXACT_ARGS_1_r01] = "_INIT_CALL_PY_EXACT_ARGS_1_r01", [_INIT_CALL_PY_EXACT_ARGS_2] = "_INIT_CALL_PY_EXACT_ARGS_2", + [_INIT_CALL_PY_EXACT_ARGS_2_r01] = "_INIT_CALL_PY_EXACT_ARGS_2_r01", [_INIT_CALL_PY_EXACT_ARGS_3] = "_INIT_CALL_PY_EXACT_ARGS_3", + [_INIT_CALL_PY_EXACT_ARGS_3_r01] = "_INIT_CALL_PY_EXACT_ARGS_3_r01", [_INIT_CALL_PY_EXACT_ARGS_4] = "_INIT_CALL_PY_EXACT_ARGS_4", + [_INIT_CALL_PY_EXACT_ARGS_4_r01] = "_INIT_CALL_PY_EXACT_ARGS_4_r01", [_INSERT_NULL] = "_INSERT_NULL", + [_INSERT_NULL_r10] = "_INSERT_NULL_r10", [_IS_NONE] = "_IS_NONE", + [_IS_NONE_r11] = "_IS_NONE_r11", [_IS_OP] = "_IS_OP", + [_IS_OP_r03] = "_IS_OP_r03", + [_IS_OP_r13] = "_IS_OP_r13", + [_IS_OP_r23] = "_IS_OP_r23", [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", + [_ITER_CHECK_LIST_r02] = "_ITER_CHECK_LIST_r02", + [_ITER_CHECK_LIST_r12] = "_ITER_CHECK_LIST_r12", + [_ITER_CHECK_LIST_r22] = "_ITER_CHECK_LIST_r22", + [_ITER_CHECK_LIST_r33] = "_ITER_CHECK_LIST_r33", [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", + [_ITER_CHECK_RANGE_r02] = "_ITER_CHECK_RANGE_r02", + [_ITER_CHECK_RANGE_r12] = "_ITER_CHECK_RANGE_r12", + [_ITER_CHECK_RANGE_r22] = "_ITER_CHECK_RANGE_r22", + [_ITER_CHECK_RANGE_r33] = "_ITER_CHECK_RANGE_r33", [_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE", + [_ITER_CHECK_TUPLE_r02] = "_ITER_CHECK_TUPLE_r02", + [_ITER_CHECK_TUPLE_r12] = "_ITER_CHECK_TUPLE_r12", + [_ITER_CHECK_TUPLE_r22] = "_ITER_CHECK_TUPLE_r22", + [_ITER_CHECK_TUPLE_r33] = "_ITER_CHECK_TUPLE_r33", + [_ITER_NEXT_INLINE] = "_ITER_NEXT_INLINE", + [_ITER_NEXT_INLINE_r23] = "_ITER_NEXT_INLINE_r23", [_ITER_NEXT_LIST_TIER_TWO] = "_ITER_NEXT_LIST_TIER_TWO", + [_ITER_NEXT_LIST_TIER_TWO_r23] = "_ITER_NEXT_LIST_TIER_TWO_r23", [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", + [_ITER_NEXT_RANGE_r03] = "_ITER_NEXT_RANGE_r03", + [_ITER_NEXT_RANGE_r13] = "_ITER_NEXT_RANGE_r13", + [_ITER_NEXT_RANGE_r23] = "_ITER_NEXT_RANGE_r23", [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", + [_ITER_NEXT_TUPLE_r03] = "_ITER_NEXT_TUPLE_r03", + [_ITER_NEXT_TUPLE_r13] = "_ITER_NEXT_TUPLE_r13", + [_ITER_NEXT_TUPLE_r23] = "_ITER_NEXT_TUPLE_r23", [_JUMP_TO_TOP] = "_JUMP_TO_TOP", + [_JUMP_TO_TOP_r00] = "_JUMP_TO_TOP_r00", [_LIST_APPEND] = "_LIST_APPEND", + [_LIST_APPEND_r10] = "_LIST_APPEND_r10", [_LIST_EXTEND] = "_LIST_EXTEND", + [_LIST_EXTEND_r11] = "_LIST_EXTEND_r11", [_LOAD_ATTR] = "_LOAD_ATTR", + [_LOAD_ATTR_r10] = "_LOAD_ATTR_r10", [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", + [_LOAD_ATTR_CLASS_r11] = "_LOAD_ATTR_CLASS_r11", + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME] = "_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME", + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11] = "_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11", [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", + [_LOAD_ATTR_INSTANCE_VALUE_r02] = "_LOAD_ATTR_INSTANCE_VALUE_r02", + [_LOAD_ATTR_INSTANCE_VALUE_r12] = "_LOAD_ATTR_INSTANCE_VALUE_r12", + [_LOAD_ATTR_INSTANCE_VALUE_r23] = "_LOAD_ATTR_INSTANCE_VALUE_r23", [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", + [_LOAD_ATTR_METHOD_LAZY_DICT_r02] = "_LOAD_ATTR_METHOD_LAZY_DICT_r02", + [_LOAD_ATTR_METHOD_LAZY_DICT_r12] = "_LOAD_ATTR_METHOD_LAZY_DICT_r12", + [_LOAD_ATTR_METHOD_LAZY_DICT_r23] = "_LOAD_ATTR_METHOD_LAZY_DICT_r23", [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", + [_LOAD_ATTR_METHOD_NO_DICT_r02] = "_LOAD_ATTR_METHOD_NO_DICT_r02", + [_LOAD_ATTR_METHOD_NO_DICT_r12] = "_LOAD_ATTR_METHOD_NO_DICT_r12", + [_LOAD_ATTR_METHOD_NO_DICT_r23] = "_LOAD_ATTR_METHOD_NO_DICT_r23", [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", + [_LOAD_ATTR_METHOD_WITH_VALUES_r02] = "_LOAD_ATTR_METHOD_WITH_VALUES_r02", + [_LOAD_ATTR_METHOD_WITH_VALUES_r12] = "_LOAD_ATTR_METHOD_WITH_VALUES_r12", + [_LOAD_ATTR_METHOD_WITH_VALUES_r23] = "_LOAD_ATTR_METHOD_WITH_VALUES_r23", [_LOAD_ATTR_MODULE] = "_LOAD_ATTR_MODULE", + [_LOAD_ATTR_MODULE_r12] = "_LOAD_ATTR_MODULE_r12", [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11", [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11", [_LOAD_ATTR_PROPERTY_FRAME] = "_LOAD_ATTR_PROPERTY_FRAME", + [_LOAD_ATTR_PROPERTY_FRAME_r01] = "_LOAD_ATTR_PROPERTY_FRAME_r01", + [_LOAD_ATTR_PROPERTY_FRAME_r11] = "_LOAD_ATTR_PROPERTY_FRAME_r11", + [_LOAD_ATTR_PROPERTY_FRAME_r22] = "_LOAD_ATTR_PROPERTY_FRAME_r22", + [_LOAD_ATTR_PROPERTY_FRAME_r33] = "_LOAD_ATTR_PROPERTY_FRAME_r33", [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", + [_LOAD_ATTR_SLOT_r02] = "_LOAD_ATTR_SLOT_r02", + [_LOAD_ATTR_SLOT_r12] = "_LOAD_ATTR_SLOT_r12", + [_LOAD_ATTR_SLOT_r23] = "_LOAD_ATTR_SLOT_r23", [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", + [_LOAD_ATTR_WITH_HINT_r12] = "_LOAD_ATTR_WITH_HINT_r12", [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", + [_LOAD_BUILD_CLASS_r01] = "_LOAD_BUILD_CLASS_r01", [_LOAD_COMMON_CONSTANT] = "_LOAD_COMMON_CONSTANT", + [_LOAD_COMMON_CONSTANT_r01] = "_LOAD_COMMON_CONSTANT_r01", + [_LOAD_COMMON_CONSTANT_r12] = "_LOAD_COMMON_CONSTANT_r12", + [_LOAD_COMMON_CONSTANT_r23] = "_LOAD_COMMON_CONSTANT_r23", [_LOAD_CONST] = "_LOAD_CONST", + [_LOAD_CONST_r01] = "_LOAD_CONST_r01", + [_LOAD_CONST_r12] = "_LOAD_CONST_r12", + [_LOAD_CONST_r23] = "_LOAD_CONST_r23", [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE", + [_LOAD_CONST_INLINE_r01] = "_LOAD_CONST_INLINE_r01", + [_LOAD_CONST_INLINE_r12] = "_LOAD_CONST_INLINE_r12", + [_LOAD_CONST_INLINE_r23] = "_LOAD_CONST_INLINE_r23", [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW", - [_LOAD_CONST_UNDER_INLINE] = "_LOAD_CONST_UNDER_INLINE", - [_LOAD_CONST_UNDER_INLINE_BORROW] = "_LOAD_CONST_UNDER_INLINE_BORROW", + [_LOAD_CONST_INLINE_BORROW_r01] = "_LOAD_CONST_INLINE_BORROW_r01", + [_LOAD_CONST_INLINE_BORROW_r12] = "_LOAD_CONST_INLINE_BORROW_r12", + [_LOAD_CONST_INLINE_BORROW_r23] = "_LOAD_CONST_INLINE_BORROW_r23", [_LOAD_DEREF] = "_LOAD_DEREF", + [_LOAD_DEREF_r01] = "_LOAD_DEREF_r01", [_LOAD_FAST] = "_LOAD_FAST", + [_LOAD_FAST_r01] = "_LOAD_FAST_r01", + [_LOAD_FAST_r12] = "_LOAD_FAST_r12", + [_LOAD_FAST_r23] = "_LOAD_FAST_r23", [_LOAD_FAST_0] = "_LOAD_FAST_0", + [_LOAD_FAST_0_r01] = "_LOAD_FAST_0_r01", + [_LOAD_FAST_0_r12] = "_LOAD_FAST_0_r12", + [_LOAD_FAST_0_r23] = "_LOAD_FAST_0_r23", [_LOAD_FAST_1] = "_LOAD_FAST_1", + [_LOAD_FAST_1_r01] = "_LOAD_FAST_1_r01", + [_LOAD_FAST_1_r12] = "_LOAD_FAST_1_r12", + [_LOAD_FAST_1_r23] = "_LOAD_FAST_1_r23", [_LOAD_FAST_2] = "_LOAD_FAST_2", + [_LOAD_FAST_2_r01] = "_LOAD_FAST_2_r01", + [_LOAD_FAST_2_r12] = "_LOAD_FAST_2_r12", + [_LOAD_FAST_2_r23] = "_LOAD_FAST_2_r23", [_LOAD_FAST_3] = "_LOAD_FAST_3", + [_LOAD_FAST_3_r01] = "_LOAD_FAST_3_r01", + [_LOAD_FAST_3_r12] = "_LOAD_FAST_3_r12", + [_LOAD_FAST_3_r23] = "_LOAD_FAST_3_r23", [_LOAD_FAST_4] = "_LOAD_FAST_4", + [_LOAD_FAST_4_r01] = "_LOAD_FAST_4_r01", + [_LOAD_FAST_4_r12] = "_LOAD_FAST_4_r12", + [_LOAD_FAST_4_r23] = "_LOAD_FAST_4_r23", [_LOAD_FAST_5] = "_LOAD_FAST_5", + [_LOAD_FAST_5_r01] = "_LOAD_FAST_5_r01", + [_LOAD_FAST_5_r12] = "_LOAD_FAST_5_r12", + [_LOAD_FAST_5_r23] = "_LOAD_FAST_5_r23", [_LOAD_FAST_6] = "_LOAD_FAST_6", + [_LOAD_FAST_6_r01] = "_LOAD_FAST_6_r01", + [_LOAD_FAST_6_r12] = "_LOAD_FAST_6_r12", + [_LOAD_FAST_6_r23] = "_LOAD_FAST_6_r23", [_LOAD_FAST_7] = "_LOAD_FAST_7", + [_LOAD_FAST_7_r01] = "_LOAD_FAST_7_r01", + [_LOAD_FAST_7_r12] = "_LOAD_FAST_7_r12", + [_LOAD_FAST_7_r23] = "_LOAD_FAST_7_r23", [_LOAD_FAST_AND_CLEAR] = "_LOAD_FAST_AND_CLEAR", + [_LOAD_FAST_AND_CLEAR_r01] = "_LOAD_FAST_AND_CLEAR_r01", + [_LOAD_FAST_AND_CLEAR_r12] = "_LOAD_FAST_AND_CLEAR_r12", + [_LOAD_FAST_AND_CLEAR_r23] = "_LOAD_FAST_AND_CLEAR_r23", [_LOAD_FAST_BORROW] = "_LOAD_FAST_BORROW", + [_LOAD_FAST_BORROW_r01] = "_LOAD_FAST_BORROW_r01", + [_LOAD_FAST_BORROW_r12] = "_LOAD_FAST_BORROW_r12", + [_LOAD_FAST_BORROW_r23] = "_LOAD_FAST_BORROW_r23", [_LOAD_FAST_BORROW_0] = "_LOAD_FAST_BORROW_0", + [_LOAD_FAST_BORROW_0_r01] = "_LOAD_FAST_BORROW_0_r01", + [_LOAD_FAST_BORROW_0_r12] = "_LOAD_FAST_BORROW_0_r12", + [_LOAD_FAST_BORROW_0_r23] = "_LOAD_FAST_BORROW_0_r23", [_LOAD_FAST_BORROW_1] = "_LOAD_FAST_BORROW_1", + [_LOAD_FAST_BORROW_1_r01] = "_LOAD_FAST_BORROW_1_r01", + [_LOAD_FAST_BORROW_1_r12] = "_LOAD_FAST_BORROW_1_r12", + [_LOAD_FAST_BORROW_1_r23] = "_LOAD_FAST_BORROW_1_r23", [_LOAD_FAST_BORROW_2] = "_LOAD_FAST_BORROW_2", + [_LOAD_FAST_BORROW_2_r01] = "_LOAD_FAST_BORROW_2_r01", + [_LOAD_FAST_BORROW_2_r12] = "_LOAD_FAST_BORROW_2_r12", + [_LOAD_FAST_BORROW_2_r23] = "_LOAD_FAST_BORROW_2_r23", [_LOAD_FAST_BORROW_3] = "_LOAD_FAST_BORROW_3", + [_LOAD_FAST_BORROW_3_r01] = "_LOAD_FAST_BORROW_3_r01", + [_LOAD_FAST_BORROW_3_r12] = "_LOAD_FAST_BORROW_3_r12", + [_LOAD_FAST_BORROW_3_r23] = "_LOAD_FAST_BORROW_3_r23", [_LOAD_FAST_BORROW_4] = "_LOAD_FAST_BORROW_4", + [_LOAD_FAST_BORROW_4_r01] = "_LOAD_FAST_BORROW_4_r01", + [_LOAD_FAST_BORROW_4_r12] = "_LOAD_FAST_BORROW_4_r12", + [_LOAD_FAST_BORROW_4_r23] = "_LOAD_FAST_BORROW_4_r23", [_LOAD_FAST_BORROW_5] = "_LOAD_FAST_BORROW_5", + [_LOAD_FAST_BORROW_5_r01] = "_LOAD_FAST_BORROW_5_r01", + [_LOAD_FAST_BORROW_5_r12] = "_LOAD_FAST_BORROW_5_r12", + [_LOAD_FAST_BORROW_5_r23] = "_LOAD_FAST_BORROW_5_r23", [_LOAD_FAST_BORROW_6] = "_LOAD_FAST_BORROW_6", + [_LOAD_FAST_BORROW_6_r01] = "_LOAD_FAST_BORROW_6_r01", + [_LOAD_FAST_BORROW_6_r12] = "_LOAD_FAST_BORROW_6_r12", + [_LOAD_FAST_BORROW_6_r23] = "_LOAD_FAST_BORROW_6_r23", [_LOAD_FAST_BORROW_7] = "_LOAD_FAST_BORROW_7", - [_LOAD_FAST_BORROW_LOAD_FAST_BORROW] = "_LOAD_FAST_BORROW_LOAD_FAST_BORROW", + [_LOAD_FAST_BORROW_7_r01] = "_LOAD_FAST_BORROW_7_r01", + [_LOAD_FAST_BORROW_7_r12] = "_LOAD_FAST_BORROW_7_r12", + [_LOAD_FAST_BORROW_7_r23] = "_LOAD_FAST_BORROW_7_r23", [_LOAD_FAST_CHECK] = "_LOAD_FAST_CHECK", - [_LOAD_FAST_LOAD_FAST] = "_LOAD_FAST_LOAD_FAST", + [_LOAD_FAST_CHECK_r01] = "_LOAD_FAST_CHECK_r01", + [_LOAD_FAST_CHECK_r12] = "_LOAD_FAST_CHECK_r12", + [_LOAD_FAST_CHECK_r23] = "_LOAD_FAST_CHECK_r23", [_LOAD_FROM_DICT_OR_DEREF] = "_LOAD_FROM_DICT_OR_DEREF", + [_LOAD_FROM_DICT_OR_DEREF_r11] = "_LOAD_FROM_DICT_OR_DEREF_r11", [_LOAD_GLOBAL] = "_LOAD_GLOBAL", + [_LOAD_GLOBAL_r00] = "_LOAD_GLOBAL_r00", [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", + [_LOAD_GLOBAL_BUILTINS_r01] = "_LOAD_GLOBAL_BUILTINS_r01", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", + [_LOAD_GLOBAL_MODULE_r01] = "_LOAD_GLOBAL_MODULE_r01", [_LOAD_LOCALS] = "_LOAD_LOCALS", + [_LOAD_LOCALS_r01] = "_LOAD_LOCALS_r01", + [_LOAD_LOCALS_r12] = "_LOAD_LOCALS_r12", + [_LOAD_LOCALS_r23] = "_LOAD_LOCALS_r23", [_LOAD_NAME] = "_LOAD_NAME", + [_LOAD_NAME_r01] = "_LOAD_NAME_r01", [_LOAD_SMALL_INT] = "_LOAD_SMALL_INT", + [_LOAD_SMALL_INT_r01] = "_LOAD_SMALL_INT_r01", + [_LOAD_SMALL_INT_r12] = "_LOAD_SMALL_INT_r12", + [_LOAD_SMALL_INT_r23] = "_LOAD_SMALL_INT_r23", [_LOAD_SMALL_INT_0] = "_LOAD_SMALL_INT_0", + [_LOAD_SMALL_INT_0_r01] = "_LOAD_SMALL_INT_0_r01", + [_LOAD_SMALL_INT_0_r12] = "_LOAD_SMALL_INT_0_r12", + [_LOAD_SMALL_INT_0_r23] = "_LOAD_SMALL_INT_0_r23", [_LOAD_SMALL_INT_1] = "_LOAD_SMALL_INT_1", + [_LOAD_SMALL_INT_1_r01] = "_LOAD_SMALL_INT_1_r01", + [_LOAD_SMALL_INT_1_r12] = "_LOAD_SMALL_INT_1_r12", + [_LOAD_SMALL_INT_1_r23] = "_LOAD_SMALL_INT_1_r23", [_LOAD_SMALL_INT_2] = "_LOAD_SMALL_INT_2", + [_LOAD_SMALL_INT_2_r01] = "_LOAD_SMALL_INT_2_r01", + [_LOAD_SMALL_INT_2_r12] = "_LOAD_SMALL_INT_2_r12", + [_LOAD_SMALL_INT_2_r23] = "_LOAD_SMALL_INT_2_r23", [_LOAD_SMALL_INT_3] = "_LOAD_SMALL_INT_3", + [_LOAD_SMALL_INT_3_r01] = "_LOAD_SMALL_INT_3_r01", + [_LOAD_SMALL_INT_3_r12] = "_LOAD_SMALL_INT_3_r12", + [_LOAD_SMALL_INT_3_r23] = "_LOAD_SMALL_INT_3_r23", [_LOAD_SPECIAL] = "_LOAD_SPECIAL", + [_LOAD_SPECIAL_r00] = "_LOAD_SPECIAL_r00", [_LOAD_SUPER_ATTR_ATTR] = "_LOAD_SUPER_ATTR_ATTR", + [_LOAD_SUPER_ATTR_ATTR_r31] = "_LOAD_SUPER_ATTR_ATTR_r31", [_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD", + [_LOAD_SUPER_ATTR_METHOD_r32] = "_LOAD_SUPER_ATTR_METHOD_r32", + [_LOCK_OBJECT] = "_LOCK_OBJECT", + [_LOCK_OBJECT_r01] = "_LOCK_OBJECT_r01", + [_LOCK_OBJECT_r11] = "_LOCK_OBJECT_r11", + [_LOCK_OBJECT_r22] = "_LOCK_OBJECT_r22", + [_LOCK_OBJECT_r33] = "_LOCK_OBJECT_r33", [_MAKE_CALLARGS_A_TUPLE] = "_MAKE_CALLARGS_A_TUPLE", + [_MAKE_CALLARGS_A_TUPLE_r33] = "_MAKE_CALLARGS_A_TUPLE_r33", [_MAKE_CELL] = "_MAKE_CELL", + [_MAKE_CELL_r00] = "_MAKE_CELL_r00", [_MAKE_FUNCTION] = "_MAKE_FUNCTION", + [_MAKE_FUNCTION_r12] = "_MAKE_FUNCTION_r12", + [_MAKE_HEAP_SAFE] = "_MAKE_HEAP_SAFE", + [_MAKE_HEAP_SAFE_r01] = "_MAKE_HEAP_SAFE_r01", + [_MAKE_HEAP_SAFE_r11] = "_MAKE_HEAP_SAFE_r11", + [_MAKE_HEAP_SAFE_r22] = "_MAKE_HEAP_SAFE_r22", + [_MAKE_HEAP_SAFE_r33] = "_MAKE_HEAP_SAFE_r33", [_MAKE_WARM] = "_MAKE_WARM", + [_MAKE_WARM_r00] = "_MAKE_WARM_r00", + [_MAKE_WARM_r11] = "_MAKE_WARM_r11", + [_MAKE_WARM_r22] = "_MAKE_WARM_r22", + [_MAKE_WARM_r33] = "_MAKE_WARM_r33", [_MAP_ADD] = "_MAP_ADD", + [_MAP_ADD_r20] = "_MAP_ADD_r20", [_MATCH_CLASS] = "_MATCH_CLASS", + [_MATCH_CLASS_r33] = "_MATCH_CLASS_r33", [_MATCH_KEYS] = "_MATCH_KEYS", + [_MATCH_KEYS_r23] = "_MATCH_KEYS_r23", [_MATCH_MAPPING] = "_MATCH_MAPPING", + [_MATCH_MAPPING_r02] = "_MATCH_MAPPING_r02", + [_MATCH_MAPPING_r12] = "_MATCH_MAPPING_r12", + [_MATCH_MAPPING_r23] = "_MATCH_MAPPING_r23", [_MATCH_SEQUENCE] = "_MATCH_SEQUENCE", + [_MATCH_SEQUENCE_r02] = "_MATCH_SEQUENCE_r02", + [_MATCH_SEQUENCE_r12] = "_MATCH_SEQUENCE_r12", + [_MATCH_SEQUENCE_r23] = "_MATCH_SEQUENCE_r23", [_MAYBE_EXPAND_METHOD] = "_MAYBE_EXPAND_METHOD", + [_MAYBE_EXPAND_METHOD_r00] = "_MAYBE_EXPAND_METHOD_r00", [_MAYBE_EXPAND_METHOD_KW] = "_MAYBE_EXPAND_METHOD_KW", + [_MAYBE_EXPAND_METHOD_KW_r11] = "_MAYBE_EXPAND_METHOD_KW_r11", [_NOP] = "_NOP", - [_POP_CALL] = "_POP_CALL", - [_POP_CALL_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_LOAD_CONST_INLINE_BORROW", - [_POP_CALL_ONE] = "_POP_CALL_ONE", - [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW", - [_POP_CALL_TWO] = "_POP_CALL_TWO", - [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW", + [_NOP_r00] = "_NOP_r00", + [_NOP_r11] = "_NOP_r11", + [_NOP_r22] = "_NOP_r22", + [_NOP_r33] = "_NOP_r33", [_POP_EXCEPT] = "_POP_EXCEPT", + [_POP_EXCEPT_r10] = "_POP_EXCEPT_r10", [_POP_ITER] = "_POP_ITER", + [_POP_ITER_r20] = "_POP_ITER_r20", [_POP_TOP] = "_POP_TOP", + [_POP_TOP_r10] = "_POP_TOP_r10", [_POP_TOP_FLOAT] = "_POP_TOP_FLOAT", + [_POP_TOP_FLOAT_r00] = "_POP_TOP_FLOAT_r00", + [_POP_TOP_FLOAT_r10] = "_POP_TOP_FLOAT_r10", + [_POP_TOP_FLOAT_r21] = "_POP_TOP_FLOAT_r21", + [_POP_TOP_FLOAT_r32] = "_POP_TOP_FLOAT_r32", [_POP_TOP_INT] = "_POP_TOP_INT", - [_POP_TOP_LOAD_CONST_INLINE] = "_POP_TOP_LOAD_CONST_INLINE", - [_POP_TOP_LOAD_CONST_INLINE_BORROW] = "_POP_TOP_LOAD_CONST_INLINE_BORROW", + [_POP_TOP_INT_r00] = "_POP_TOP_INT_r00", + [_POP_TOP_INT_r10] = "_POP_TOP_INT_r10", + [_POP_TOP_INT_r21] = "_POP_TOP_INT_r21", + [_POP_TOP_INT_r32] = "_POP_TOP_INT_r32", [_POP_TOP_NOP] = "_POP_TOP_NOP", + [_POP_TOP_NOP_r00] = "_POP_TOP_NOP_r00", + [_POP_TOP_NOP_r10] = "_POP_TOP_NOP_r10", + [_POP_TOP_NOP_r21] = "_POP_TOP_NOP_r21", + [_POP_TOP_NOP_r32] = "_POP_TOP_NOP_r32", + [_POP_TOP_OPARG] = "_POP_TOP_OPARG", + [_POP_TOP_OPARG_r00] = "_POP_TOP_OPARG_r00", [_POP_TOP_UNICODE] = "_POP_TOP_UNICODE", - [_POP_TWO] = "_POP_TWO", - [_POP_TWO_LOAD_CONST_INLINE_BORROW] = "_POP_TWO_LOAD_CONST_INLINE_BORROW", + [_POP_TOP_UNICODE_r00] = "_POP_TOP_UNICODE_r00", + [_POP_TOP_UNICODE_r10] = "_POP_TOP_UNICODE_r10", + [_POP_TOP_UNICODE_r21] = "_POP_TOP_UNICODE_r21", + [_POP_TOP_UNICODE_r32] = "_POP_TOP_UNICODE_r32", [_PUSH_EXC_INFO] = "_PUSH_EXC_INFO", + [_PUSH_EXC_INFO_r02] = "_PUSH_EXC_INFO_r02", + [_PUSH_EXC_INFO_r12] = "_PUSH_EXC_INFO_r12", + [_PUSH_EXC_INFO_r23] = "_PUSH_EXC_INFO_r23", [_PUSH_FRAME] = "_PUSH_FRAME", + [_PUSH_FRAME_r10] = "_PUSH_FRAME_r10", [_PUSH_NULL] = "_PUSH_NULL", + [_PUSH_NULL_r01] = "_PUSH_NULL_r01", + [_PUSH_NULL_r12] = "_PUSH_NULL_r12", + [_PUSH_NULL_r23] = "_PUSH_NULL_r23", [_PUSH_NULL_CONDITIONAL] = "_PUSH_NULL_CONDITIONAL", + [_PUSH_NULL_CONDITIONAL_r00] = "_PUSH_NULL_CONDITIONAL_r00", + [_PUSH_TAGGED_ZERO] = "_PUSH_TAGGED_ZERO", + [_PUSH_TAGGED_ZERO_r01] = "_PUSH_TAGGED_ZERO_r01", + [_PUSH_TAGGED_ZERO_r12] = "_PUSH_TAGGED_ZERO_r12", + [_PUSH_TAGGED_ZERO_r23] = "_PUSH_TAGGED_ZERO_r23", + [_PY_FRAME_EX] = "_PY_FRAME_EX", + [_PY_FRAME_EX_r31] = "_PY_FRAME_EX_r31", [_PY_FRAME_GENERAL] = "_PY_FRAME_GENERAL", + [_PY_FRAME_GENERAL_r01] = "_PY_FRAME_GENERAL_r01", [_PY_FRAME_KW] = "_PY_FRAME_KW", + [_PY_FRAME_KW_r11] = "_PY_FRAME_KW_r11", + [_RECORD_3OS_GEN_FUNC] = "_RECORD_3OS_GEN_FUNC", + [_RECORD_4OS] = "_RECORD_4OS", + [_RECORD_BOUND_METHOD] = "_RECORD_BOUND_METHOD", + [_RECORD_CALLABLE] = "_RECORD_CALLABLE", + [_RECORD_CALLABLE_KW] = "_RECORD_CALLABLE_KW", + [_RECORD_CODE] = "_RECORD_CODE", + [_RECORD_NOS] = "_RECORD_NOS", + [_RECORD_NOS_GEN_FUNC] = "_RECORD_NOS_GEN_FUNC", + [_RECORD_NOS_TYPE] = "_RECORD_NOS_TYPE", + [_RECORD_TOS] = "_RECORD_TOS", + [_RECORD_TOS_TYPE] = "_RECORD_TOS_TYPE", [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE", + [_REPLACE_WITH_TRUE_r02] = "_REPLACE_WITH_TRUE_r02", + [_REPLACE_WITH_TRUE_r12] = "_REPLACE_WITH_TRUE_r12", + [_REPLACE_WITH_TRUE_r23] = "_REPLACE_WITH_TRUE_r23", [_RESUME_CHECK] = "_RESUME_CHECK", + [_RESUME_CHECK_r00] = "_RESUME_CHECK_r00", + [_RESUME_CHECK_r11] = "_RESUME_CHECK_r11", + [_RESUME_CHECK_r22] = "_RESUME_CHECK_r22", + [_RESUME_CHECK_r33] = "_RESUME_CHECK_r33", [_RETURN_GENERATOR] = "_RETURN_GENERATOR", + [_RETURN_GENERATOR_r01] = "_RETURN_GENERATOR_r01", [_RETURN_VALUE] = "_RETURN_VALUE", + [_RETURN_VALUE_r11] = "_RETURN_VALUE_r11", + [_RROT_3] = "_RROT_3", + [_RROT_3_r03] = "_RROT_3_r03", + [_RROT_3_r13] = "_RROT_3_r13", + [_RROT_3_r23] = "_RROT_3_r23", + [_RROT_3_r33] = "_RROT_3_r33", [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", + [_SAVE_RETURN_OFFSET_r00] = "_SAVE_RETURN_OFFSET_r00", + [_SAVE_RETURN_OFFSET_r11] = "_SAVE_RETURN_OFFSET_r11", + [_SAVE_RETURN_OFFSET_r22] = "_SAVE_RETURN_OFFSET_r22", + [_SAVE_RETURN_OFFSET_r33] = "_SAVE_RETURN_OFFSET_r33", + [_SEND_ASYNC_GEN_TIER_TWO] = "_SEND_ASYNC_GEN_TIER_TWO", + [_SEND_ASYNC_GEN_TIER_TWO_r33] = "_SEND_ASYNC_GEN_TIER_TWO_r33", [_SEND_GEN_FRAME] = "_SEND_GEN_FRAME", + [_SEND_GEN_FRAME_r33] = "_SEND_GEN_FRAME_r33", + [_SEND_VIRTUAL_TIER_TWO] = "_SEND_VIRTUAL_TIER_TWO", + [_SEND_VIRTUAL_TIER_TWO_r03] = "_SEND_VIRTUAL_TIER_TWO_r03", + [_SEND_VIRTUAL_TIER_TWO_r13] = "_SEND_VIRTUAL_TIER_TWO_r13", + [_SEND_VIRTUAL_TIER_TWO_r23] = "_SEND_VIRTUAL_TIER_TWO_r23", + [_SEND_VIRTUAL_TIER_TWO_r33] = "_SEND_VIRTUAL_TIER_TWO_r33", [_SETUP_ANNOTATIONS] = "_SETUP_ANNOTATIONS", + [_SETUP_ANNOTATIONS_r00] = "_SETUP_ANNOTATIONS_r00", [_SET_ADD] = "_SET_ADD", + [_SET_ADD_r10] = "_SET_ADD_r10", [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", + [_SET_FUNCTION_ATTRIBUTE_r01] = "_SET_FUNCTION_ATTRIBUTE_r01", + [_SET_FUNCTION_ATTRIBUTE_r11] = "_SET_FUNCTION_ATTRIBUTE_r11", + [_SET_FUNCTION_ATTRIBUTE_r21] = "_SET_FUNCTION_ATTRIBUTE_r21", + [_SET_FUNCTION_ATTRIBUTE_r32] = "_SET_FUNCTION_ATTRIBUTE_r32", [_SET_IP] = "_SET_IP", + [_SET_IP_r00] = "_SET_IP_r00", + [_SET_IP_r11] = "_SET_IP_r11", + [_SET_IP_r22] = "_SET_IP_r22", + [_SET_IP_r33] = "_SET_IP_r33", [_SET_UPDATE] = "_SET_UPDATE", + [_SET_UPDATE_r11] = "_SET_UPDATE_r11", + [_SPILL_OR_RELOAD] = "_SPILL_OR_RELOAD", + [_SPILL_OR_RELOAD_r01] = "_SPILL_OR_RELOAD_r01", + [_SPILL_OR_RELOAD_r02] = "_SPILL_OR_RELOAD_r02", + [_SPILL_OR_RELOAD_r03] = "_SPILL_OR_RELOAD_r03", + [_SPILL_OR_RELOAD_r10] = "_SPILL_OR_RELOAD_r10", + [_SPILL_OR_RELOAD_r12] = "_SPILL_OR_RELOAD_r12", + [_SPILL_OR_RELOAD_r13] = "_SPILL_OR_RELOAD_r13", + [_SPILL_OR_RELOAD_r20] = "_SPILL_OR_RELOAD_r20", + [_SPILL_OR_RELOAD_r21] = "_SPILL_OR_RELOAD_r21", + [_SPILL_OR_RELOAD_r23] = "_SPILL_OR_RELOAD_r23", + [_SPILL_OR_RELOAD_r30] = "_SPILL_OR_RELOAD_r30", + [_SPILL_OR_RELOAD_r31] = "_SPILL_OR_RELOAD_r31", + [_SPILL_OR_RELOAD_r32] = "_SPILL_OR_RELOAD_r32", [_START_EXECUTOR] = "_START_EXECUTOR", + [_START_EXECUTOR_r00] = "_START_EXECUTOR_r00", [_STORE_ATTR] = "_STORE_ATTR", + [_STORE_ATTR_r20] = "_STORE_ATTR_r20", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", + [_STORE_ATTR_INSTANCE_VALUE_r21] = "_STORE_ATTR_INSTANCE_VALUE_r21", [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", + [_STORE_ATTR_SLOT_r21] = "_STORE_ATTR_SLOT_r21", [_STORE_ATTR_WITH_HINT] = "_STORE_ATTR_WITH_HINT", + [_STORE_ATTR_WITH_HINT_r21] = "_STORE_ATTR_WITH_HINT_r21", [_STORE_DEREF] = "_STORE_DEREF", - [_STORE_FAST] = "_STORE_FAST", - [_STORE_FAST_0] = "_STORE_FAST_0", - [_STORE_FAST_1] = "_STORE_FAST_1", - [_STORE_FAST_2] = "_STORE_FAST_2", - [_STORE_FAST_3] = "_STORE_FAST_3", - [_STORE_FAST_4] = "_STORE_FAST_4", - [_STORE_FAST_5] = "_STORE_FAST_5", - [_STORE_FAST_6] = "_STORE_FAST_6", - [_STORE_FAST_7] = "_STORE_FAST_7", - [_STORE_FAST_LOAD_FAST] = "_STORE_FAST_LOAD_FAST", - [_STORE_FAST_STORE_FAST] = "_STORE_FAST_STORE_FAST", + [_STORE_DEREF_r10] = "_STORE_DEREF_r10", [_STORE_GLOBAL] = "_STORE_GLOBAL", + [_STORE_GLOBAL_r10] = "_STORE_GLOBAL_r10", [_STORE_NAME] = "_STORE_NAME", + [_STORE_NAME_r10] = "_STORE_NAME_r10", [_STORE_SLICE] = "_STORE_SLICE", + [_STORE_SLICE_r30] = "_STORE_SLICE_r30", [_STORE_SUBSCR] = "_STORE_SUBSCR", + [_STORE_SUBSCR_r30] = "_STORE_SUBSCR_r30", [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", + [_STORE_SUBSCR_DICT_r31] = "_STORE_SUBSCR_DICT_r31", + [_STORE_SUBSCR_DICT_KNOWN_HASH] = "_STORE_SUBSCR_DICT_KNOWN_HASH", + [_STORE_SUBSCR_DICT_KNOWN_HASH_r31] = "_STORE_SUBSCR_DICT_KNOWN_HASH_r31", [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", + [_STORE_SUBSCR_LIST_INT_r32] = "_STORE_SUBSCR_LIST_INT_r32", [_SWAP] = "_SWAP", + [_SWAP_r11] = "_SWAP_r11", [_SWAP_2] = "_SWAP_2", + [_SWAP_2_r02] = "_SWAP_2_r02", + [_SWAP_2_r12] = "_SWAP_2_r12", + [_SWAP_2_r22] = "_SWAP_2_r22", + [_SWAP_2_r33] = "_SWAP_2_r33", [_SWAP_3] = "_SWAP_3", + [_SWAP_3_r03] = "_SWAP_3_r03", + [_SWAP_3_r13] = "_SWAP_3_r13", + [_SWAP_3_r23] = "_SWAP_3_r23", + [_SWAP_3_r33] = "_SWAP_3_r33", + [_SWAP_FAST] = "_SWAP_FAST", + [_SWAP_FAST_r01] = "_SWAP_FAST_r01", + [_SWAP_FAST_r11] = "_SWAP_FAST_r11", + [_SWAP_FAST_r22] = "_SWAP_FAST_r22", + [_SWAP_FAST_r33] = "_SWAP_FAST_r33", + [_SWAP_FAST_0] = "_SWAP_FAST_0", + [_SWAP_FAST_0_r01] = "_SWAP_FAST_0_r01", + [_SWAP_FAST_0_r11] = "_SWAP_FAST_0_r11", + [_SWAP_FAST_0_r22] = "_SWAP_FAST_0_r22", + [_SWAP_FAST_0_r33] = "_SWAP_FAST_0_r33", + [_SWAP_FAST_1] = "_SWAP_FAST_1", + [_SWAP_FAST_1_r01] = "_SWAP_FAST_1_r01", + [_SWAP_FAST_1_r11] = "_SWAP_FAST_1_r11", + [_SWAP_FAST_1_r22] = "_SWAP_FAST_1_r22", + [_SWAP_FAST_1_r33] = "_SWAP_FAST_1_r33", + [_SWAP_FAST_2] = "_SWAP_FAST_2", + [_SWAP_FAST_2_r01] = "_SWAP_FAST_2_r01", + [_SWAP_FAST_2_r11] = "_SWAP_FAST_2_r11", + [_SWAP_FAST_2_r22] = "_SWAP_FAST_2_r22", + [_SWAP_FAST_2_r33] = "_SWAP_FAST_2_r33", + [_SWAP_FAST_3] = "_SWAP_FAST_3", + [_SWAP_FAST_3_r01] = "_SWAP_FAST_3_r01", + [_SWAP_FAST_3_r11] = "_SWAP_FAST_3_r11", + [_SWAP_FAST_3_r22] = "_SWAP_FAST_3_r22", + [_SWAP_FAST_3_r33] = "_SWAP_FAST_3_r33", + [_SWAP_FAST_4] = "_SWAP_FAST_4", + [_SWAP_FAST_4_r01] = "_SWAP_FAST_4_r01", + [_SWAP_FAST_4_r11] = "_SWAP_FAST_4_r11", + [_SWAP_FAST_4_r22] = "_SWAP_FAST_4_r22", + [_SWAP_FAST_4_r33] = "_SWAP_FAST_4_r33", + [_SWAP_FAST_5] = "_SWAP_FAST_5", + [_SWAP_FAST_5_r01] = "_SWAP_FAST_5_r01", + [_SWAP_FAST_5_r11] = "_SWAP_FAST_5_r11", + [_SWAP_FAST_5_r22] = "_SWAP_FAST_5_r22", + [_SWAP_FAST_5_r33] = "_SWAP_FAST_5_r33", + [_SWAP_FAST_6] = "_SWAP_FAST_6", + [_SWAP_FAST_6_r01] = "_SWAP_FAST_6_r01", + [_SWAP_FAST_6_r11] = "_SWAP_FAST_6_r11", + [_SWAP_FAST_6_r22] = "_SWAP_FAST_6_r22", + [_SWAP_FAST_6_r33] = "_SWAP_FAST_6_r33", + [_SWAP_FAST_7] = "_SWAP_FAST_7", + [_SWAP_FAST_7_r01] = "_SWAP_FAST_7_r01", + [_SWAP_FAST_7_r11] = "_SWAP_FAST_7_r11", + [_SWAP_FAST_7_r22] = "_SWAP_FAST_7_r22", + [_SWAP_FAST_7_r33] = "_SWAP_FAST_7_r33", [_TIER2_RESUME_CHECK] = "_TIER2_RESUME_CHECK", + [_TIER2_RESUME_CHECK_r00] = "_TIER2_RESUME_CHECK_r00", + [_TIER2_RESUME_CHECK_r11] = "_TIER2_RESUME_CHECK_r11", + [_TIER2_RESUME_CHECK_r22] = "_TIER2_RESUME_CHECK_r22", + [_TIER2_RESUME_CHECK_r33] = "_TIER2_RESUME_CHECK_r33", [_TO_BOOL] = "_TO_BOOL", + [_TO_BOOL_r11] = "_TO_BOOL_r11", [_TO_BOOL_BOOL] = "_TO_BOOL_BOOL", + [_TO_BOOL_BOOL_r01] = "_TO_BOOL_BOOL_r01", + [_TO_BOOL_BOOL_r11] = "_TO_BOOL_BOOL_r11", + [_TO_BOOL_BOOL_r22] = "_TO_BOOL_BOOL_r22", + [_TO_BOOL_BOOL_r33] = "_TO_BOOL_BOOL_r33", [_TO_BOOL_INT] = "_TO_BOOL_INT", + [_TO_BOOL_INT_r02] = "_TO_BOOL_INT_r02", + [_TO_BOOL_INT_r12] = "_TO_BOOL_INT_r12", + [_TO_BOOL_INT_r23] = "_TO_BOOL_INT_r23", [_TO_BOOL_LIST] = "_TO_BOOL_LIST", + [_TO_BOOL_LIST_r02] = "_TO_BOOL_LIST_r02", + [_TO_BOOL_LIST_r12] = "_TO_BOOL_LIST_r12", + [_TO_BOOL_LIST_r23] = "_TO_BOOL_LIST_r23", [_TO_BOOL_NONE] = "_TO_BOOL_NONE", + [_TO_BOOL_NONE_r01] = "_TO_BOOL_NONE_r01", + [_TO_BOOL_NONE_r11] = "_TO_BOOL_NONE_r11", + [_TO_BOOL_NONE_r22] = "_TO_BOOL_NONE_r22", + [_TO_BOOL_NONE_r33] = "_TO_BOOL_NONE_r33", [_TO_BOOL_STR] = "_TO_BOOL_STR", + [_TO_BOOL_STR_r02] = "_TO_BOOL_STR_r02", + [_TO_BOOL_STR_r12] = "_TO_BOOL_STR_r12", + [_TO_BOOL_STR_r23] = "_TO_BOOL_STR_r23", [_UNARY_INVERT] = "_UNARY_INVERT", + [_UNARY_INVERT_r12] = "_UNARY_INVERT_r12", [_UNARY_NEGATIVE] = "_UNARY_NEGATIVE", + [_UNARY_NEGATIVE_r12] = "_UNARY_NEGATIVE_r12", + [_UNARY_NEGATIVE_FLOAT_INPLACE] = "_UNARY_NEGATIVE_FLOAT_INPLACE", + [_UNARY_NEGATIVE_FLOAT_INPLACE_r02] = "_UNARY_NEGATIVE_FLOAT_INPLACE_r02", + [_UNARY_NEGATIVE_FLOAT_INPLACE_r12] = "_UNARY_NEGATIVE_FLOAT_INPLACE_r12", + [_UNARY_NEGATIVE_FLOAT_INPLACE_r23] = "_UNARY_NEGATIVE_FLOAT_INPLACE_r23", [_UNARY_NOT] = "_UNARY_NOT", + [_UNARY_NOT_r01] = "_UNARY_NOT_r01", + [_UNARY_NOT_r11] = "_UNARY_NOT_r11", + [_UNARY_NOT_r22] = "_UNARY_NOT_r22", + [_UNARY_NOT_r33] = "_UNARY_NOT_r33", [_UNPACK_EX] = "_UNPACK_EX", + [_UNPACK_EX_r10] = "_UNPACK_EX_r10", [_UNPACK_SEQUENCE] = "_UNPACK_SEQUENCE", + [_UNPACK_SEQUENCE_r10] = "_UNPACK_SEQUENCE_r10", [_UNPACK_SEQUENCE_LIST] = "_UNPACK_SEQUENCE_LIST", + [_UNPACK_SEQUENCE_LIST_r10] = "_UNPACK_SEQUENCE_LIST_r10", [_UNPACK_SEQUENCE_TUPLE] = "_UNPACK_SEQUENCE_TUPLE", + [_UNPACK_SEQUENCE_TUPLE_r10] = "_UNPACK_SEQUENCE_TUPLE_r10", [_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE", + [_UNPACK_SEQUENCE_TWO_TUPLE_r12] = "_UNPACK_SEQUENCE_TWO_TUPLE_r12", + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE] = "_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE", + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03] = "_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03", + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13] = "_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13", + [_UNPACK_SEQUENCE_UNIQUE_TUPLE] = "_UNPACK_SEQUENCE_UNIQUE_TUPLE", + [_UNPACK_SEQUENCE_UNIQUE_TUPLE_r10] = "_UNPACK_SEQUENCE_UNIQUE_TUPLE_r10", + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE] = "_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE", + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02] = "_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02", + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12] = "_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12", + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23] = "_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23", [_WITH_EXCEPT_START] = "_WITH_EXCEPT_START", + [_WITH_EXCEPT_START_r33] = "_WITH_EXCEPT_START_r33", [_YIELD_VALUE] = "_YIELD_VALUE", + [_YIELD_VALUE_r11] = "_YIELD_VALUE_r11", }; int _PyUop_num_popped(int opcode, int oparg) { @@ -717,10 +6253,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_FAST_AND_CLEAR: return 0; - case _LOAD_FAST_LOAD_FAST: - return 0; - case _LOAD_FAST_BORROW_LOAD_FAST_BORROW: - return 0; case _LOAD_CONST: return 0; case _LOAD_SMALL_INT_0: @@ -733,28 +6265,24 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_SMALL_INT: return 0; - case _STORE_FAST_0: - return 1; - case _STORE_FAST_1: + case _SWAP_FAST_0: return 1; - case _STORE_FAST_2: + case _SWAP_FAST_1: return 1; - case _STORE_FAST_3: + case _SWAP_FAST_2: return 1; - case _STORE_FAST_4: + case _SWAP_FAST_3: return 1; - case _STORE_FAST_5: + case _SWAP_FAST_4: return 1; - case _STORE_FAST_6: + case _SWAP_FAST_5: return 1; - case _STORE_FAST_7: + case _SWAP_FAST_6: return 1; - case _STORE_FAST: + case _SWAP_FAST_7: return 1; - case _STORE_FAST_LOAD_FAST: + case _SWAP_FAST: return 1; - case _STORE_FAST_STORE_FAST: - return 2; case _POP_TOP: return 1; case _POP_TOP_NOP: @@ -765,8 +6293,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _POP_TOP_UNICODE: return 1; - case _POP_TWO: - return 2; + case _POP_TOP_OPARG: + return oparg; case _PUSH_NULL: return 0; case _END_FOR: @@ -774,9 +6302,11 @@ int _PyUop_num_popped(int opcode, int oparg) case _POP_ITER: return 2; case _END_SEND: - return 2; + return 3; case _UNARY_NEGATIVE: return 1; + case _UNARY_NEGATIVE_FLOAT_INPLACE: + return 1; case _UNARY_NOT: return 1; case _TO_BOOL: @@ -795,6 +6325,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _TO_BOOL_NONE: return 1; + case _GUARD_NOS_COMPACT_ASCII: + return 0; case _GUARD_NOS_UNICODE: return 0; case _GUARD_TOS_UNICODE: @@ -819,6 +6351,18 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBTRACT_INT: return 2; + case _BINARY_OP_ADD_INT_INPLACE: + return 2; + case _BINARY_OP_SUBTRACT_INT_INPLACE: + return 2; + case _BINARY_OP_MULTIPLY_INT_INPLACE: + return 2; + case _BINARY_OP_ADD_INT_INPLACE_RIGHT: + return 2; + case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT: + return 2; + case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT: + return 2; case _GUARD_NOS_FLOAT: return 0; case _GUARD_TOS_FLOAT: @@ -829,16 +6373,32 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBTRACT_FLOAT: return 2; - case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: + case _BINARY_OP_ADD_FLOAT_INPLACE: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE: + return 2; + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE: return 2; - case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: + case _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT: return 2; - case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT: + return 2; + case _BINARY_OP_TRUEDIV_FLOAT: + return 2; + case _BINARY_OP_TRUEDIV_FLOAT_INPLACE: + return 2; + case _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT: return 2; case _BINARY_OP_ADD_UNICODE: return 2; case _BINARY_OP_INPLACE_ADD_UNICODE: return 2; + case _GUARD_BINARY_OP_EXTEND_LHS: + return 0; + case _GUARD_BINARY_OP_EXTEND_RHS: + return 0; case _GUARD_BINARY_OP_EXTEND: return 0; case _BINARY_OP_EXTEND: @@ -853,16 +6413,28 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBSCR_STR_INT: return 2; + case _BINARY_OP_SUBSCR_USTR_INT: + return 2; case _GUARD_NOS_TUPLE: return 0; case _GUARD_TOS_TUPLE: return 0; + case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS: + return 0; case _BINARY_OP_SUBSCR_TUPLE_INT: return 2; - case _GUARD_NOS_DICT: + case _GUARD_NOS_DICT_SUBSCRIPT: + return 0; + case _GUARD_NOS_DICT_STORE_SUBSCRIPT: + return 0; + case _GUARD_TOS_ANY_DICT: return 0; case _GUARD_TOS_DICT: return 0; + case _GUARD_TOS_FROZENDICT: + return 0; + case _BINARY_OP_SUBSCR_DICT_KNOWN_HASH: + return 2; case _BINARY_OP_SUBSCR_DICT: return 2; case _BINARY_OP_SUBSCR_CHECK_FUNC: @@ -879,12 +6451,16 @@ int _PyUop_num_popped(int opcode, int oparg) return 3; case _STORE_SUBSCR_DICT: return 3; + case _STORE_SUBSCR_DICT_KNOWN_HASH: + return 3; case _DELETE_SUBSCR: return 2; case _CALL_INTRINSIC_1: return 1; case _CALL_INTRINSIC_2: return 2; + case _MAKE_HEAP_SAFE: + return 0; case _RETURN_VALUE: return 1; case _GET_AITER: @@ -895,6 +6471,16 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _SEND_GEN_FRAME: return 1; + case _GUARD_TOS_IS_NONE: + return 0; + case _GUARD_NOS_NOT_NULL: + return 0; + case _SEND_VIRTUAL_TIER_TWO: + return 1; + case _GUARD_3OS_ASYNC_GEN_ASEND: + return 0; + case _SEND_ASYNC_GEN_TIER_TWO: + return 3; case _YIELD_VALUE: return 1; case _POP_EXCEPT: @@ -911,8 +6497,14 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _UNPACK_SEQUENCE_TWO_TUPLE: return 1; + case _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE: + return 1; + case _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE: + return 1; case _UNPACK_SEQUENCE_TUPLE: return 1; + case _UNPACK_SEQUENCE_UNIQUE_TUPLE: + return 1; case _UNPACK_SEQUENCE_LIST: return 1; case _UNPACK_EX: @@ -981,13 +6573,19 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _LOAD_SUPER_ATTR_ATTR: return 3; + case _GUARD_NOS_TYPE_VERSION: + return 0; + case _GUARD_LOAD_SUPER_ATTR_METHOD: + return 0; case _LOAD_SUPER_ATTR_METHOD: return 3; case _LOAD_ATTR: return 1; case _GUARD_TYPE_VERSION: return 0; - case _GUARD_TYPE_VERSION_AND_LOCK: + case _GUARD_TYPE_VERSION_LOCKED: + return 0; + case _GUARD_TYPE: return 0; case _CHECK_MANAGED_OBJECT_HAS_VALUES: return 0; @@ -1005,10 +6603,14 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _LOAD_ATTR_PROPERTY_FRAME: return 1; + case _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME: + return 1; case _GUARD_DORV_NO_DICT: return 0; case _STORE_ATTR_INSTANCE_VALUE: return 2; + case _LOCK_OBJECT: + return 0; case _STORE_ATTR_WITH_HINT: return 2; case _STORE_ATTR_SLOT: @@ -1027,6 +6629,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _GUARD_TOS_ANY_SET: return 0; + case _GUARD_TOS_SET: + return 0; + case _GUARD_TOS_FROZENSET: + return 0; case _CONTAINS_OP_SET: return 2; case _CONTAINS_OP_DICT: @@ -1053,10 +6659,26 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GET_ITER: return 1; - case _GET_YIELD_FROM_ITER: + case _GUARD_ITERATOR: + return 0; + case _GUARD_ITER_VIRTUAL: + return 0; + case _PUSH_TAGGED_ZERO: + return 0; + case _GET_ITER_TRAD: return 1; case _FOR_ITER_TIER_TWO: return 0; + case _GUARD_TYPE_ITER: + return 0; + case _ITER_NEXT_INLINE: + return 0; + case _GUARD_NOS_ITER_VIRTUAL: + return 0; + case _GUARD_TOS_NOT_NULL: + return 0; + case _FOR_ITER_VIRTUAL_TIER_TWO: + return 0; case _ITER_CHECK_LIST: return 0; case _GUARD_NOT_EXHAUSTED_LIST: @@ -1087,8 +6709,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: return 0; - case _GUARD_KEYS_VERSION: - return 0; case _LOAD_ATTR_METHOD_WITH_VALUES: return 1; case _LOAD_ATTR_METHOD_NO_DICT: @@ -1145,8 +6765,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _GUARD_NOS_NULL: return 0; - case _GUARD_NOS_NOT_NULL: - return 0; case _GUARD_THIRD_NULL: return 0; case _GUARD_CALLABLE_TYPE_1: @@ -1161,20 +6779,30 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _CALL_TUPLE_1: return 3; - case _CHECK_AND_ALLOCATE_OBJECT: + case _CHECK_OBJECT: + return 0; + case _ALLOCATE_OBJECT: return 0; case _CREATE_INIT_FRAME: return 2 + oparg; case _EXIT_INIT_CHECK: return 1; + case _GUARD_CALLABLE_BUILTIN_CLASS: + return 0; case _CALL_BUILTIN_CLASS: - return 2 + oparg; + return 0; + case _GUARD_CALLABLE_BUILTIN_O: + return 0; case _CALL_BUILTIN_O: return 2 + oparg; + case _GUARD_CALLABLE_BUILTIN_FAST: + return 0; case _CALL_BUILTIN_FAST: - return 2 + oparg; + return 0; + case _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS: + return 0; case _CALL_BUILTIN_FAST_WITH_KEYWORDS: - return 2 + oparg; + return 0; case _GUARD_CALLABLE_LEN: return 0; case _CALL_LEN: @@ -1187,14 +6815,32 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _CALL_LIST_APPEND: return 3; + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_O: + return 0; case _CALL_METHOD_DESCRIPTOR_O: return 2 + oparg; + case _CHECK_RECURSION_LIMIT: + return 0; + case _CALL_METHOD_DESCRIPTOR_O_INLINE: + return 1 + oparg; + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: + return 0; case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: - return 2 + oparg; + return 0; + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE: + return 0; + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS: + return 0; case _CALL_METHOD_DESCRIPTOR_NOARGS: return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE: + return 1 + oparg; + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST: + return 0; case _CALL_METHOD_DESCRIPTOR_FAST: - return 2 + oparg; + return 0; + case _CALL_METHOD_DESCRIPTOR_FAST_INLINE: + return 0; case _MAYBE_EXPAND_METHOD_KW: return 0; case _PY_FRAME_KW: @@ -1211,6 +6857,14 @@ int _PyUop_num_popped(int opcode, int oparg) return 3 + oparg; case _MAKE_CALLARGS_A_TUPLE: return 0; + case _CHECK_IS_PY_CALLABLE_EX: + return 0; + case _PY_FRAME_EX: + return 4; + case _CHECK_IS_NOT_PY_CALLABLE_EX: + return 0; + case _CALL_FUNCTION_EX_NON_PY_GENERAL: + return 4; case _MAKE_FUNCTION: return 1; case _SET_FUNCTION_ATTRIBUTE: @@ -1245,6 +6899,26 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _GUARD_IS_FALSE_POP: return 1; + case _GUARD_BIT_IS_SET_POP_4: + return 1; + case _GUARD_BIT_IS_SET_POP_5: + return 1; + case _GUARD_BIT_IS_SET_POP_6: + return 1; + case _GUARD_BIT_IS_SET_POP_7: + return 1; + case _GUARD_BIT_IS_SET_POP: + return 1; + case _GUARD_BIT_IS_UNSET_POP_4: + return 1; + case _GUARD_BIT_IS_UNSET_POP_5: + return 1; + case _GUARD_BIT_IS_UNSET_POP_6: + return 1; + case _GUARD_BIT_IS_UNSET_POP_7: + return 1; + case _GUARD_BIT_IS_UNSET_POP: + return 1; case _GUARD_IS_NONE_POP: return 1; case _GUARD_IS_NOT_NONE_POP: @@ -1259,35 +6933,15 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _EXIT_TRACE: return 0; + case _DYNAMIC_EXIT: + return 0; case _CHECK_VALIDITY: return 0; case _LOAD_CONST_INLINE: return 0; - case _POP_TOP_LOAD_CONST_INLINE: - return 1; case _LOAD_CONST_INLINE_BORROW: return 0; - case _POP_CALL: - return 2; - case _POP_CALL_ONE: - return 3; - case _POP_CALL_TWO: - return 4; - case _POP_TOP_LOAD_CONST_INLINE_BORROW: - return 1; - case _POP_TWO_LOAD_CONST_INLINE_BORROW: - return 2; - case _POP_CALL_LOAD_CONST_INLINE_BORROW: - return 2; - case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: - return 3; - case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: - return 4; - case _LOAD_CONST_UNDER_INLINE: - return 1; - case _LOAD_CONST_UNDER_INLINE_BORROW: - return 1; - case _CHECK_FUNCTION: + case _RROT_3: return 0; case _START_EXECUTOR: return 0; @@ -1297,10 +6951,56 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _DEOPT: return 0; + case _HANDLE_PENDING_AND_DEOPT: + return 0; case _ERROR_POP_N: return 0; + case _SPILL_OR_RELOAD: + return 0; case _TIER2_RESUME_CHECK: return 0; + case _COLD_EXIT: + return 0; + case _COLD_DYNAMIC_EXIT: + return 0; + case _GUARD_CODE_VERSION__PUSH_FRAME: + return 0; + case _GUARD_CODE_VERSION_YIELD_VALUE: + return 0; + case _GUARD_CODE_VERSION_RETURN_VALUE: + return 0; + case _GUARD_CODE_VERSION_RETURN_GENERATOR: + return 0; + case _GUARD_IP__PUSH_FRAME: + return 0; + case _GUARD_IP_YIELD_VALUE: + return 0; + case _GUARD_IP_RETURN_VALUE: + return 0; + case _GUARD_IP_RETURN_GENERATOR: + return 0; + case _RECORD_TOS: + return 0; + case _RECORD_TOS_TYPE: + return 0; + case _RECORD_NOS: + return 0; + case _RECORD_NOS_TYPE: + return 0; + case _RECORD_NOS_GEN_FUNC: + return 0; + case _RECORD_3OS_GEN_FUNC: + return 0; + case _RECORD_4OS: + return 0; + case _RECORD_CALLABLE: + return 0; + case _RECORD_CALLABLE_KW: + return 0; + case _RECORD_BOUND_METHOD: + return 0; + case _RECORD_CODE: + return 0; default: return -1; } diff --git a/Include/longobject.h b/Include/longobject.h index 19f06977036d056..38673bc18785fab 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -166,6 +166,44 @@ PyAPI_FUNC(PyObject *) PyLong_FromString(const char *, char **, int); PyAPI_FUNC(unsigned long) PyOS_strtoul(const char *, char **, int); PyAPI_FUNC(long) PyOS_strtol(const char *, char **, int); +/* --- Import/Export API -------------------------------------------------- */ + +typedef struct PyLongLayout { + uint8_t bits_per_digit; + uint8_t digit_size; + int8_t digits_order; + int8_t digit_endianness; +} PyLongLayout; + +PyAPI_FUNC(const PyLongLayout*) PyLong_GetNativeLayout(void); + +typedef struct PyLongExport { + int64_t value; + uint8_t negative; + Py_ssize_t ndigits; + const void *digits; + // Member used internally, must not be used for other purpose. + Py_uintptr_t _reserved; +} PyLongExport; + +PyAPI_FUNC(int) PyLong_Export( + PyObject *obj, + PyLongExport *export_long); +PyAPI_FUNC(void) PyLong_FreeExport( + PyLongExport *export_long); + + +/* --- PyLongWriter API --------------------------------------------------- */ + +typedef struct PyLongWriter PyLongWriter; + +PyAPI_FUNC(PyLongWriter*) PyLongWriter_Create( + int negative, + Py_ssize_t ndigits, + void **digits); +PyAPI_FUNC(PyObject*) PyLongWriter_Finish(PyLongWriter *writer); +PyAPI_FUNC(void) PyLongWriter_Discard(PyLongWriter *writer); + #ifndef Py_LIMITED_API # define Py_CPYTHON_LONGOBJECT_H # include "cpython/longobject.h" diff --git a/Include/marshal.h b/Include/marshal.h index f773587bdd0429e..2ccb112b40c7527 100644 --- a/Include/marshal.h +++ b/Include/marshal.h @@ -1,31 +1,18 @@ - /* Interface for marshal.c */ #ifndef Py_MARSHAL_H #define Py_MARSHAL_H -#ifndef Py_LIMITED_API - #ifdef __cplusplus extern "C" { #endif -PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, - Py_ssize_t); -PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int); - -#define Py_MARSHAL_VERSION 5 - -PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *); -PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *); -PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromFile(FILE *); -PyAPI_FUNC(PyObject *) PyMarshal_ReadLastObjectFromFile(FILE *); - -PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int); -PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int); +#ifndef Py_LIMITED_API +# define _Py_CPYTHON_MARSHAL_H +# include "cpython/marshal.h" +# undef _Py_CPYTHON_MARSHAL_H +#endif #ifdef __cplusplus } #endif - -#endif /* Py_LIMITED_API */ #endif /* !Py_MARSHAL_H */ diff --git a/Include/modsupport.h b/Include/modsupport.h index af995f567b004c9..cb47ad8cd2727fd 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -56,58 +56,6 @@ PyAPI_FUNC(int) PyModule_ExecDef(PyObject *module, PyModuleDef *def); #define Py_CLEANUP_SUPPORTED 0x20000 -#define PYTHON_API_VERSION 1013 -#define PYTHON_API_STRING "1013" -/* The API version is maintained (independently from the Python version) - so we can detect mismatches between the interpreter and dynamically - loaded modules. These are diagnosed by an error message but - the module is still loaded (because the mismatch can only be tested - after loading the module). The error message is intended to - explain the core dump a few seconds later. - - The symbol PYTHON_API_STRING defines the same value as a string - literal. *** PLEASE MAKE SURE THE DEFINITIONS MATCH. *** - - Please add a line or two to the top of this log for each API - version change: - - 22-Feb-2006 MvL 1013 PEP 353 - long indices for sequence lengths - - 19-Aug-2002 GvR 1012 Changes to string object struct for - interning changes, saving 3 bytes. - - 17-Jul-2001 GvR 1011 Descr-branch, just to be on the safe side - - 25-Jan-2001 FLD 1010 Parameters added to PyCode_New() and - PyFrame_New(); Python 2.1a2 - - 14-Mar-2000 GvR 1009 Unicode API added - - 3-Jan-1999 GvR 1007 Decided to change back! (Don't reuse 1008!) - - 3-Dec-1998 GvR 1008 Python 1.5.2b1 - - 18-Jan-1997 GvR 1007 string interning and other speedups - - 11-Oct-1996 GvR renamed Py_Ellipses to Py_Ellipsis :-( - - 30-Jul-1996 GvR Slice and ellipses syntax added - - 23-Jul-1996 GvR For 1.4 -- better safe than sorry this time :-) - - 7-Nov-1995 GvR Keyword arguments (should've been done at 1.3 :-( ) - - 10-Jan-1995 GvR Renamed globals to new naming scheme - - 9-Jan-1995 GvR Initial version (incompatible with older API) -*/ - -/* The PYTHON_ABI_VERSION is introduced in PEP 384. For the lifetime of - Python 3, it will stay at the value of 3; changes to the limited API - must be performed in a strictly backwards-compatible manner. */ -#define PYTHON_ABI_VERSION 3 -#define PYTHON_ABI_STRING "3" - PyAPI_FUNC(PyObject *) PyModule_Create2(PyModuleDef*, int apiver); #ifdef Py_LIMITED_API @@ -134,6 +82,72 @@ PyAPI_FUNC(PyObject *) PyModule_FromDefAndSpec2(PyModuleDef *def, #endif /* New in 3.5 */ +/* ABI info & checking (new in 3.15) */ +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +typedef struct PyABIInfo { + uint8_t abiinfo_major_version; + uint8_t abiinfo_minor_version; + uint16_t flags; + uint32_t build_version; + uint32_t abi_version; +} PyABIInfo; +#define PyABIInfo_STABLE 0x0001 +#define PyABIInfo_GIL 0x0002 +#define PyABIInfo_FREETHREADED 0x0004 +#define PyABIInfo_INTERNAL 0x0008 + +#define PyABIInfo_FREETHREADING_AGNOSTIC (PyABIInfo_GIL|PyABIInfo_FREETHREADED) + +PyAPI_FUNC(int) PyABIInfo_Check(PyABIInfo *info, const char *module_name); + +// Define the defaults +#ifdef Py_LIMITED_API + #define _PyABIInfo_DEFAULT_FLAG_STABLE PyABIInfo_STABLE + #if Py_LIMITED_API == 3 + #define PyABIInfo_DEFAULT_ABI_VERSION _Py_PACK_VERSION(3, 2) + #else + #define PyABIInfo_DEFAULT_ABI_VERSION Py_LIMITED_API + #endif +#else + #define _PyABIInfo_DEFAULT_FLAG_STABLE 0 + #define PyABIInfo_DEFAULT_ABI_VERSION PY_VERSION_HEX +#endif +#if defined(Py_LIMITED_API) && defined(_Py_OPAQUE_PYOBJECT) + #define _PyABIInfo_DEFAULT_FLAG_FT PyABIInfo_FREETHREADING_AGNOSTIC +#elif defined(Py_GIL_DISABLED) + #define _PyABIInfo_DEFAULT_FLAG_FT PyABIInfo_FREETHREADED +#else + #define _PyABIInfo_DEFAULT_FLAG_FT PyABIInfo_GIL +#endif +#if defined(Py_BUILD_CORE) + #define _PyABIInfo_DEFAULT_FLAG_INTERNAL PyABIInfo_INTERNAL +#else + #define _PyABIInfo_DEFAULT_FLAG_INTERNAL 0 +#endif + +#define PyABIInfo_DEFAULT_FLAGS ( \ + _PyABIInfo_DEFAULT_FLAG_STABLE \ + | _PyABIInfo_DEFAULT_FLAG_FT \ + | _PyABIInfo_DEFAULT_FLAG_INTERNAL \ + ) \ + ///////////////////////////////////////////////////////// + +#define _PyABIInfo_DEFAULT { \ + 1, 0, \ + PyABIInfo_DEFAULT_FLAGS, \ + PY_VERSION_HEX, \ + PyABIInfo_DEFAULT_ABI_VERSION } \ + ///////////////////////////////////////////////////////// + +#define PyABIInfo_VAR(NAME) \ + static PyABIInfo NAME = _PyABIInfo_DEFAULT; + +#undef _PyABIInfo_DEFAULT_STABLE +#undef _PyABIInfo_DEFAULT_FT +#undef _PyABIInfo_DEFAULT_INTERNAL + +#endif /* ABI info (new in 3.15) */ + #ifndef Py_LIMITED_API # define Py_CPYTHON_MODSUPPORT_H # include "cpython/modsupport.h" diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 17634a93f8fa6f5..88c66672ff164a8 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -73,20 +73,6 @@ struct PyModuleDef_Slot { void *value; }; -#define Py_mod_create 1 -#define Py_mod_exec 2 -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030c0000 -# define Py_mod_multiple_interpreters 3 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 -# define Py_mod_gil 4 -#endif - - -#ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 4 -#endif - #endif /* New in 3.5 */ /* for Py_mod_multiple_interpreters: */ @@ -102,10 +88,21 @@ struct PyModuleDef_Slot { # define Py_MOD_GIL_NOT_USED ((void *)1) #endif -#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +#if !defined(Py_LIMITED_API) +# if defined(Py_GIL_DISABLED) PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil); +# endif #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(PyObject *) PyModule_FromSlotsAndSpec(const PySlot *slots, + PyObject *spec); +PyAPI_FUNC(int) PyModule_Exec(PyObject *module); +PyAPI_FUNC(int) PyModule_GetStateSize(PyObject *module, Py_ssize_t *result); +PyAPI_FUNC(int) PyModule_GetToken(PyObject *module, void **result); +PyAPI_FUNC(void*) PyModule_GetState_DuringGC(PyObject*); +PyAPI_FUNC(int) PyModule_GetToken_DuringGC(PyObject *module, void **result); +#endif #ifndef _Py_OPAQUE_PYOBJECT struct PyModuleDef { diff --git a/Include/monitoring.h b/Include/monitoring.h deleted file mode 100644 index 985f7f230e44e3d..000000000000000 --- a/Include/monitoring.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef Py_MONITORING_H -#define Py_MONITORING_H -#ifdef __cplusplus -extern "C" { -#endif - -// There is currently no limited API for monitoring - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_MONITORING_H -# include "cpython/monitoring.h" -# undef Py_CPYTHON_MONITORING_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_MONITORING_H */ diff --git a/Include/object.h b/Include/object.h index b1bcc9481871b4c..20c2dab4401fef0 100644 --- a/Include/object.h +++ b/Include/object.h @@ -71,6 +71,8 @@ whose size is determined when the object is allocated. * * Statically allocated objects might be shared between * interpreters, so must be marked as immortal. + * + * Before changing this, see the check in PyModuleDef_Init(). */ #if defined(Py_GIL_DISABLED) #define PyObject_HEAD_INIT(type) \ @@ -123,20 +125,9 @@ whose size is determined when the object is allocated. /* PyObject is opaque */ #elif !defined(Py_GIL_DISABLED) struct _object { -#if (defined(__GNUC__) || defined(__clang__)) \ - && !(defined __STDC_VERSION__ && __STDC_VERSION__ >= 201112L) - // On C99 and older, anonymous union is a GCC and clang extension - __extension__ -#endif -#ifdef _MSC_VER - // Ignore MSC warning C4201: "nonstandard extension used: - // nameless struct/union" - __pragma(warning(push)) - __pragma(warning(disable: 4201)) -#endif - union { + _Py_ANONYMOUS union { #if SIZEOF_VOID_P > 4 - PY_INT64_T ob_refcnt_full; /* This field is needed for efficient initialization with Clang on ARM */ + int64_t ob_refcnt_full; /* This field is needed for efficient initialization with Clang on ARM */ struct { # if PY_BIG_ENDIAN uint16_t ob_flags; @@ -149,15 +140,12 @@ struct _object { # endif }; #else - Py_ssize_t ob_refcnt; + Py_ssize_t ob_refcnt; // part of stable ABI; do not change #endif _Py_ALIGNED_DEF(_PyObject_MIN_ALIGNMENT, char) _aligner; }; -#ifdef _MSC_VER - __pragma(warning(pop)) -#endif - PyTypeObject *ob_type; + PyTypeObject *ob_type; // part of stable ABI; do not change }; #else // Objects that are not owned by any thread use a thread id (tid) of zero. @@ -185,7 +173,7 @@ struct _object { #ifndef _Py_OPAQUE_PYOBJECT struct PyVarObject { PyObject ob_base; - Py_ssize_t ob_size; /* Number of items in variable part */ + Py_ssize_t ob_size; // Number of items in variable part. Part of stable ABI }; #endif typedef struct PyVarObject PyVarObject; @@ -198,135 +186,72 @@ typedef struct PyVarObject PyVarObject; PyAPI_FUNC(int) Py_Is(PyObject *x, PyObject *y); #define Py_Is(x, y) ((x) == (y)) -#if defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) -PyAPI_FUNC(uintptr_t) _Py_GetThreadLocal_Addr(void); +PyAPI_DATA(PyTypeObject) PyLong_Type; +PyAPI_DATA(PyTypeObject) PyBool_Type; -static inline uintptr_t -_Py_ThreadId(void) -{ - uintptr_t tid; -#if defined(_MSC_VER) && defined(_M_X64) - tid = __readgsqword(48); -#elif defined(_MSC_VER) && defined(_M_IX86) - tid = __readfsdword(24); -#elif defined(_MSC_VER) && defined(_M_ARM64) - tid = __getReg(18); -#elif defined(__MINGW32__) && defined(_M_X64) - tid = __readgsqword(48); -#elif defined(__MINGW32__) && defined(_M_IX86) - tid = __readfsdword(24); -#elif defined(__MINGW32__) && defined(_M_ARM64) - tid = __getReg(18); -#elif defined(__i386__) - __asm__("movl %%gs:0, %0" : "=r" (tid)); // 32-bit always uses GS -#elif defined(__MACH__) && defined(__x86_64__) - __asm__("movq %%gs:0, %0" : "=r" (tid)); // x86_64 macOSX uses GS -#elif defined(__x86_64__) - __asm__("movq %%fs:0, %0" : "=r" (tid)); // x86_64 Linux, BSD uses FS -#elif defined(__arm__) && __ARM_ARCH >= 7 - __asm__ ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tid)); -#elif defined(__aarch64__) && defined(__APPLE__) - __asm__ ("mrs %0, tpidrro_el0" : "=r" (tid)); -#elif defined(__aarch64__) - __asm__ ("mrs %0, tpidr_el0" : "=r" (tid)); -#elif defined(__powerpc64__) - #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) - tid = (uintptr_t)__builtin_thread_pointer(); - #else - // r13 is reserved for use as system thread ID by the Power 64-bit ABI. - register uintptr_t tp __asm__ ("r13"); - __asm__("" : "=r" (tp)); - tid = tp; - #endif -#elif defined(__powerpc__) - #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) - tid = (uintptr_t)__builtin_thread_pointer(); - #else - // r2 is reserved for use as system thread ID by the Power 32-bit ABI. - register uintptr_t tp __asm__ ("r2"); - __asm__ ("" : "=r" (tp)); - tid = tp; - #endif -#elif defined(__s390__) && defined(__GNUC__) - // Both GCC and Clang have supported __builtin_thread_pointer - // for s390 from long time ago. - tid = (uintptr_t)__builtin_thread_pointer(); -#elif defined(__riscv) - #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) - tid = (uintptr_t)__builtin_thread_pointer(); - #else - // tp is Thread Pointer provided by the RISC-V ABI. - __asm__ ("mv %0, tp" : "=r" (tid)); - #endif -#else - // Fallback to a portable implementation if we do not have a faster - // platform-specific implementation. - tid = _Py_GetThreadLocal_Addr(); +/* Definitions for the stable ABI */ +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 14) +PyAPI_FUNC(PyTypeObject*) Py_TYPE(PyObject *ob); +#endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(Py_ssize_t) Py_SIZE(PyObject *ob); +PyAPI_FUNC(int) Py_IS_TYPE(PyObject *ob, PyTypeObject *type); +PyAPI_FUNC(void) Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size); #endif - return tid; -} -static inline Py_ALWAYS_INLINE int -_Py_IsOwnedByCurrentThread(PyObject *ob) +#ifndef _Py_OPAQUE_PYOBJECT + +static inline void +Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { -#ifdef _Py_THREAD_SANITIZER - return _Py_atomic_load_uintptr_relaxed(&ob->ob_tid) == _Py_ThreadId(); -#else - return ob->ob_tid == _Py_ThreadId(); -#endif + ob->ob_type = type; } -#endif - -// Py_TYPE() implementation for the stable ABI -PyAPI_FUNC(PyTypeObject*) Py_TYPE(PyObject *ob); -#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030e0000 - // Stable ABI implements Py_TYPE() as a function call - // on limited C API version 3.14 and newer. +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < _Py_PACK_VERSION(3, 11) +// Non-limited API & limited API 3.11 & below: use static inline functions and +// use _PyObject_CAST so that users don't need their own casts +# define Py_TYPE(ob) _Py_TYPE_impl(_PyObject_CAST(ob)) +# define Py_SIZE(ob) _Py_SIZE_impl(_PyObject_CAST(ob)) +# define Py_IS_TYPE(ob, type) _Py_IS_TYPE_impl(_PyObject_CAST(ob), (type)) +# define Py_SET_SIZE(ob, size) _Py_SET_SIZE_impl(_PyVarObject_CAST(ob), (size)) +# define Py_SET_TYPE(ob, type) Py_SET_TYPE(_PyObject_CAST(ob), type) +#elif Py_LIMITED_API+0 < _Py_PACK_VERSION(3, 15) +// Limited API 3.11-3.14: use static inline functions, without casts +# define Py_SIZE(ob) _Py_SIZE_impl(ob) +# define Py_IS_TYPE(ob, type) _Py_IS_TYPE_impl((ob), (type)) +# define Py_SET_SIZE(ob, size) _Py_SET_SIZE_impl((ob), (size)) +# if Py_LIMITED_API+0 < _Py_PACK_VERSION(3, 14) +// Py_TYPE() is static inline only on Limited API 3.13 and below +# define Py_TYPE(ob) _Py_TYPE_impl(ob) +# endif #else - static inline PyTypeObject* _Py_TYPE(PyObject *ob) - { - return ob->ob_type; - } - #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 - # define Py_TYPE(ob) _Py_TYPE(_PyObject_CAST(ob)) - #else - # define Py_TYPE(ob) _Py_TYPE(ob) - #endif +// Limited API 3.15+: use function calls #endif -PyAPI_DATA(PyTypeObject) PyLong_Type; -PyAPI_DATA(PyTypeObject) PyBool_Type; +static inline +PyTypeObject* _Py_TYPE_impl(PyObject *ob) +{ + return ob->ob_type; +} -#ifndef _Py_OPAQUE_PYOBJECT // bpo-39573: The Py_SET_SIZE() function must be used to set an object size. -static inline Py_ssize_t Py_SIZE(PyObject *ob) { +static inline Py_ssize_t +_Py_SIZE_impl(PyObject *ob) +{ assert(Py_TYPE(ob) != &PyLong_Type); assert(Py_TYPE(ob) != &PyBool_Type); return _PyVarObject_CAST(ob)->ob_size; } -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SIZE(ob) Py_SIZE(_PyObject_CAST(ob)) -#endif -#endif // !defined(_Py_OPAQUE_PYOBJECT) -static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { +static inline int +_Py_IS_TYPE_impl(PyObject *ob, PyTypeObject *type) +{ return Py_TYPE(ob) == type; } -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_IS_TYPE(ob, type) Py_IS_TYPE(_PyObject_CAST(ob), (type)) -#endif - - -#ifndef _Py_OPAQUE_PYOBJECT -static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { - ob->ob_type = type; -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SET_TYPE(ob, type) Py_SET_TYPE(_PyObject_CAST(ob), type) -#endif -static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { +static inline void +_Py_SET_SIZE_impl(PyVarObject *ob, Py_ssize_t size) +{ assert(Py_TYPE(_PyObject_CAST(ob)) != &PyLong_Type); assert(Py_TYPE(_PyObject_CAST(ob)) != &PyBool_Type); #ifdef Py_GIL_DISABLED @@ -335,9 +260,7 @@ static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { ob->ob_size = size; #endif } -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SET_SIZE(ob, size) Py_SET_SIZE(_PyVarObject_CAST(ob), (size)) -#endif + #endif // !defined(_Py_OPAQUE_PYOBJECT) @@ -383,6 +306,11 @@ typedef Py_hash_t (*hashfunc)(PyObject *); typedef PyObject *(*richcmpfunc) (PyObject *, PyObject *, int); typedef PyObject *(*getiterfunc) (PyObject *); typedef PyObject *(*iternextfunc) (PyObject *); +typedef struct { + PyObject *object; + Py_ssize_t index; +} _PyObjectIndexPair; +typedef _PyObjectIndexPair (*_Py_iteritemfunc) (PyObject *, Py_ssize_t index); typedef PyObject *(*descrgetfunc) (PyObject *, PyObject *, PyObject *); typedef int (*descrsetfunc) (PyObject *, PyObject *, PyObject *); typedef int (*initproc)(PyObject *, PyObject *, PyObject *); @@ -436,6 +364,9 @@ PyAPI_FUNC(Py_ssize_t) PyType_GetTypeDataSize(PyTypeObject *cls); PyAPI_FUNC(int) PyType_GetBaseByToken(PyTypeObject *, void *, PyTypeObject **); #define Py_TP_USE_SPEC NULL #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(PyObject *) PyType_FromSlots(struct PySlot *slots); +#endif /* Generic type check */ PyAPI_FUNC(int) PyType_IsSubtype(PyTypeObject *, PyTypeObject *); @@ -541,7 +472,7 @@ given type object has a specified feature. #define Py_TPFLAGS_INLINE_VALUES (1 << 2) /* Placement of weakref pointers are managed by the VM, not by the type. - * The VM will automatically set tp_weaklistoffset. + * The VM will automatically set tp_weaklistoffset. Implies Py_TPFLAGS_HAVE_GC. */ #define Py_TPFLAGS_MANAGED_WEAKREF (1 << 3) @@ -550,6 +481,9 @@ given type object has a specified feature. */ #define Py_TPFLAGS_MANAGED_DICT (1 << 4) +/* Type has dictionary or weakref pointers that are managed by VM and has + * to allocate space to store these. + */ #define Py_TPFLAGS_PREHEADER (Py_TPFLAGS_MANAGED_WEAKREF | Py_TPFLAGS_MANAGED_DICT) /* Set if instances of the type object are treated as sequences for pattern matching */ @@ -645,9 +579,12 @@ given type object has a specified feature. // Flag values for ob_flags (16 bits available, if SIZEOF_VOID_P > 4). #define _Py_IMMORTAL_FLAGS (1 << 0) +#define _Py_LEGACY_ABI_CHECK_FLAG (1 << 1) /* see PyModuleDef_Init() */ #define _Py_STATICALLY_ALLOCATED_FLAG (1 << 2) -#if defined(Py_GIL_DISABLED) && defined(Py_DEBUG) -#define _Py_TYPE_REVEALED_FLAG (1 << 3) +#if !defined(Py_LIMITED_API) +# if defined(Py_GIL_DISABLED) && defined(Py_DEBUG) +# define _Py_TYPE_REVEALED_FLAG (1 << 3) +# endif #endif #define Py_CONSTANT_NONE 0 @@ -703,8 +640,13 @@ PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */ # define Py_NotImplemented (&_Py_NotImplementedStruct) #endif -/* Macro for returning Py_NotImplemented from a function */ -#define Py_RETURN_NOTIMPLEMENTED return Py_NotImplemented +/* Macro for returning Py_NotImplemented from a function. Only treat + * Py_NotImplemented as immortal in the limited C API 3.12 and newer. */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030c0000 +# define Py_RETURN_NOTIMPLEMENTED return Py_NewRef(Py_NotImplemented) +#else +# define Py_RETURN_NOTIMPLEMENTED return Py_NotImplemented +#endif /* Rich comparison opcodes */ #define Py_LT 0 @@ -842,6 +784,20 @@ PyAPI_FUNC(PyObject *) PyType_GetModuleByDef(PyTypeObject *, PyModuleDef *); PyAPI_FUNC(int) PyType_Freeze(PyTypeObject *type); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *); +PyAPI_FUNC(PyObject *) PyType_GetModuleByToken(PyTypeObject *type, + const void *token); +PyAPI_FUNC(void *) PyObject_GetTypeData_DuringGC(PyObject *obj, + PyTypeObject *cls); +PyAPI_FUNC(void *) PyType_GetModuleState_DuringGC(PyTypeObject *); +PyAPI_FUNC(int) PyType_GetBaseByToken_DuringGC(PyTypeObject *, + void *, PyTypeObject **); +PyAPI_FUNC(PyObject *) PyType_GetModule_DuringGC(PyTypeObject *); +PyAPI_FUNC(PyObject *) PyType_GetModuleByToken_DuringGC(PyTypeObject *type, + const void *token); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 1d5c74adefcd35d..758542720acf317 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -26,111 +26,110 @@ extern "C" { #define FORMAT_WITH_SPEC 13 #define GET_AITER 14 #define GET_ANEXT 15 -#define GET_ITER 16 +#define GET_LEN 16 #define RESERVED 17 -#define GET_LEN 18 -#define GET_YIELD_FROM_ITER 19 -#define INTERPRETER_EXIT 20 -#define LOAD_BUILD_CLASS 21 -#define LOAD_LOCALS 22 -#define MAKE_FUNCTION 23 -#define MATCH_KEYS 24 -#define MATCH_MAPPING 25 -#define MATCH_SEQUENCE 26 -#define NOP 27 -#define NOT_TAKEN 28 -#define POP_EXCEPT 29 -#define POP_ITER 30 -#define POP_TOP 31 -#define PUSH_EXC_INFO 32 -#define PUSH_NULL 33 -#define RETURN_GENERATOR 34 -#define RETURN_VALUE 35 -#define SETUP_ANNOTATIONS 36 -#define STORE_SLICE 37 -#define STORE_SUBSCR 38 -#define TO_BOOL 39 -#define UNARY_INVERT 40 -#define UNARY_NEGATIVE 41 -#define UNARY_NOT 42 -#define WITH_EXCEPT_START 43 -#define BINARY_OP 44 -#define BUILD_INTERPOLATION 45 -#define BUILD_LIST 46 -#define BUILD_MAP 47 -#define BUILD_SET 48 -#define BUILD_SLICE 49 -#define BUILD_STRING 50 -#define BUILD_TUPLE 51 -#define CALL 52 -#define CALL_INTRINSIC_1 53 -#define CALL_INTRINSIC_2 54 -#define CALL_KW 55 -#define COMPARE_OP 56 -#define CONTAINS_OP 57 -#define CONVERT_VALUE 58 -#define COPY 59 -#define COPY_FREE_VARS 60 -#define DELETE_ATTR 61 -#define DELETE_DEREF 62 -#define DELETE_FAST 63 -#define DELETE_GLOBAL 64 -#define DELETE_NAME 65 -#define DICT_MERGE 66 -#define DICT_UPDATE 67 -#define END_ASYNC_FOR 68 -#define EXTENDED_ARG 69 -#define FOR_ITER 70 -#define GET_AWAITABLE 71 -#define IMPORT_FROM 72 -#define IMPORT_NAME 73 -#define IS_OP 74 -#define JUMP_BACKWARD 75 -#define JUMP_BACKWARD_NO_INTERRUPT 76 -#define JUMP_FORWARD 77 -#define LIST_APPEND 78 -#define LIST_EXTEND 79 -#define LOAD_ATTR 80 -#define LOAD_COMMON_CONSTANT 81 -#define LOAD_CONST 82 -#define LOAD_DEREF 83 -#define LOAD_FAST 84 -#define LOAD_FAST_AND_CLEAR 85 -#define LOAD_FAST_BORROW 86 -#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 87 -#define LOAD_FAST_CHECK 88 -#define LOAD_FAST_LOAD_FAST 89 -#define LOAD_FROM_DICT_OR_DEREF 90 -#define LOAD_FROM_DICT_OR_GLOBALS 91 -#define LOAD_GLOBAL 92 -#define LOAD_NAME 93 -#define LOAD_SMALL_INT 94 -#define LOAD_SPECIAL 95 -#define LOAD_SUPER_ATTR 96 -#define MAKE_CELL 97 -#define MAP_ADD 98 -#define MATCH_CLASS 99 -#define POP_JUMP_IF_FALSE 100 -#define POP_JUMP_IF_NONE 101 -#define POP_JUMP_IF_NOT_NONE 102 -#define POP_JUMP_IF_TRUE 103 -#define RAISE_VARARGS 104 -#define RERAISE 105 -#define SEND 106 -#define SET_ADD 107 -#define SET_FUNCTION_ATTRIBUTE 108 -#define SET_UPDATE 109 -#define STORE_ATTR 110 -#define STORE_DEREF 111 -#define STORE_FAST 112 -#define STORE_FAST_LOAD_FAST 113 -#define STORE_FAST_STORE_FAST 114 -#define STORE_GLOBAL 115 -#define STORE_NAME 116 -#define SWAP 117 -#define UNPACK_EX 118 -#define UNPACK_SEQUENCE 119 -#define YIELD_VALUE 120 +#define INTERPRETER_EXIT 18 +#define LOAD_BUILD_CLASS 19 +#define LOAD_LOCALS 20 +#define MAKE_FUNCTION 21 +#define MATCH_KEYS 22 +#define MATCH_MAPPING 23 +#define MATCH_SEQUENCE 24 +#define NOP 25 +#define NOT_TAKEN 26 +#define POP_EXCEPT 27 +#define POP_ITER 28 +#define POP_TOP 29 +#define PUSH_EXC_INFO 30 +#define PUSH_NULL 31 +#define RETURN_GENERATOR 32 +#define RETURN_VALUE 33 +#define SETUP_ANNOTATIONS 34 +#define STORE_SLICE 35 +#define STORE_SUBSCR 36 +#define TO_BOOL 37 +#define UNARY_INVERT 38 +#define UNARY_NEGATIVE 39 +#define UNARY_NOT 40 +#define WITH_EXCEPT_START 41 +#define BINARY_OP 42 +#define BUILD_INTERPOLATION 43 +#define BUILD_LIST 44 +#define BUILD_MAP 45 +#define BUILD_SET 46 +#define BUILD_SLICE 47 +#define BUILD_STRING 48 +#define BUILD_TUPLE 49 +#define CALL 50 +#define CALL_INTRINSIC_1 51 +#define CALL_INTRINSIC_2 52 +#define CALL_KW 53 +#define COMPARE_OP 54 +#define CONTAINS_OP 55 +#define CONVERT_VALUE 56 +#define COPY 57 +#define COPY_FREE_VARS 58 +#define DELETE_ATTR 59 +#define DELETE_DEREF 60 +#define DELETE_FAST 61 +#define DELETE_GLOBAL 62 +#define DELETE_NAME 63 +#define DICT_MERGE 64 +#define DICT_UPDATE 65 +#define END_ASYNC_FOR 66 +#define EXTENDED_ARG 67 +#define FOR_ITER 68 +#define GET_AWAITABLE 69 +#define GET_ITER 70 +#define IMPORT_FROM 71 +#define IMPORT_NAME 72 +#define IS_OP 73 +#define JUMP_BACKWARD 74 +#define JUMP_BACKWARD_NO_INTERRUPT 75 +#define JUMP_FORWARD 76 +#define LIST_APPEND 77 +#define LIST_EXTEND 78 +#define LOAD_ATTR 79 +#define LOAD_COMMON_CONSTANT 80 +#define LOAD_CONST 81 +#define LOAD_DEREF 82 +#define LOAD_FAST 83 +#define LOAD_FAST_AND_CLEAR 84 +#define LOAD_FAST_BORROW 85 +#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 86 +#define LOAD_FAST_CHECK 87 +#define LOAD_FAST_LOAD_FAST 88 +#define LOAD_FROM_DICT_OR_DEREF 89 +#define LOAD_FROM_DICT_OR_GLOBALS 90 +#define LOAD_GLOBAL 91 +#define LOAD_NAME 92 +#define LOAD_SMALL_INT 93 +#define LOAD_SPECIAL 94 +#define LOAD_SUPER_ATTR 95 +#define MAKE_CELL 96 +#define MAP_ADD 97 +#define MATCH_CLASS 98 +#define POP_JUMP_IF_FALSE 99 +#define POP_JUMP_IF_NONE 100 +#define POP_JUMP_IF_NOT_NONE 101 +#define POP_JUMP_IF_TRUE 102 +#define RAISE_VARARGS 103 +#define RERAISE 104 +#define SEND 105 +#define SET_ADD 106 +#define SET_FUNCTION_ATTRIBUTE 107 +#define SET_UPDATE 108 +#define STORE_ATTR 109 +#define STORE_DEREF 110 +#define STORE_FAST 111 +#define STORE_FAST_LOAD_FAST 112 +#define STORE_FAST_STORE_FAST 113 +#define STORE_GLOBAL 114 +#define STORE_NAME 115 +#define SWAP 116 +#define UNPACK_EX 117 +#define UNPACK_SEQUENCE 118 +#define YIELD_VALUE 119 #define RESUME 128 #define BINARY_OP_ADD_FLOAT 129 #define BINARY_OP_ADD_INT 130 @@ -144,97 +143,107 @@ extern "C" { #define BINARY_OP_SUBSCR_LIST_SLICE 138 #define BINARY_OP_SUBSCR_STR_INT 139 #define BINARY_OP_SUBSCR_TUPLE_INT 140 -#define BINARY_OP_SUBTRACT_FLOAT 141 -#define BINARY_OP_SUBTRACT_INT 142 -#define CALL_ALLOC_AND_ENTER_INIT 143 -#define CALL_BOUND_METHOD_EXACT_ARGS 144 -#define CALL_BOUND_METHOD_GENERAL 145 -#define CALL_BUILTIN_CLASS 146 -#define CALL_BUILTIN_FAST 147 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 148 -#define CALL_BUILTIN_O 149 -#define CALL_ISINSTANCE 150 -#define CALL_KW_BOUND_METHOD 151 -#define CALL_KW_NON_PY 152 -#define CALL_KW_PY 153 -#define CALL_LEN 154 -#define CALL_LIST_APPEND 155 -#define CALL_METHOD_DESCRIPTOR_FAST 156 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 157 -#define CALL_METHOD_DESCRIPTOR_NOARGS 158 -#define CALL_METHOD_DESCRIPTOR_O 159 -#define CALL_NON_PY_GENERAL 160 -#define CALL_PY_EXACT_ARGS 161 -#define CALL_PY_GENERAL 162 -#define CALL_STR_1 163 -#define CALL_TUPLE_1 164 -#define CALL_TYPE_1 165 -#define COMPARE_OP_FLOAT 166 -#define COMPARE_OP_INT 167 -#define COMPARE_OP_STR 168 -#define CONTAINS_OP_DICT 169 -#define CONTAINS_OP_SET 170 -#define FOR_ITER_GEN 171 -#define FOR_ITER_LIST 172 -#define FOR_ITER_RANGE 173 -#define FOR_ITER_TUPLE 174 -#define JUMP_BACKWARD_JIT 175 -#define JUMP_BACKWARD_NO_JIT 176 -#define LOAD_ATTR_CLASS 177 -#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 178 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 179 -#define LOAD_ATTR_INSTANCE_VALUE 180 -#define LOAD_ATTR_METHOD_LAZY_DICT 181 -#define LOAD_ATTR_METHOD_NO_DICT 182 -#define LOAD_ATTR_METHOD_WITH_VALUES 183 -#define LOAD_ATTR_MODULE 184 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 185 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 186 -#define LOAD_ATTR_PROPERTY 187 -#define LOAD_ATTR_SLOT 188 -#define LOAD_ATTR_WITH_HINT 189 -#define LOAD_GLOBAL_BUILTIN 190 -#define LOAD_GLOBAL_MODULE 191 -#define LOAD_SUPER_ATTR_ATTR 192 -#define LOAD_SUPER_ATTR_METHOD 193 -#define RESUME_CHECK 194 -#define SEND_GEN 195 -#define STORE_ATTR_INSTANCE_VALUE 196 -#define STORE_ATTR_SLOT 197 -#define STORE_ATTR_WITH_HINT 198 -#define STORE_SUBSCR_DICT 199 -#define STORE_SUBSCR_LIST_INT 200 -#define TO_BOOL_ALWAYS_TRUE 201 -#define TO_BOOL_BOOL 202 -#define TO_BOOL_INT 203 -#define TO_BOOL_LIST 204 -#define TO_BOOL_NONE 205 -#define TO_BOOL_STR 206 -#define UNPACK_SEQUENCE_LIST 207 -#define UNPACK_SEQUENCE_TUPLE 208 -#define UNPACK_SEQUENCE_TWO_TUPLE 209 -#define INSTRUMENTED_END_FOR 234 -#define INSTRUMENTED_POP_ITER 235 -#define INSTRUMENTED_END_SEND 236 -#define INSTRUMENTED_FOR_ITER 237 -#define INSTRUMENTED_INSTRUCTION 238 -#define INSTRUMENTED_JUMP_FORWARD 239 -#define INSTRUMENTED_NOT_TAKEN 240 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 241 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 242 -#define INSTRUMENTED_POP_JUMP_IF_NONE 243 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 244 -#define INSTRUMENTED_RESUME 245 -#define INSTRUMENTED_RETURN_VALUE 246 -#define INSTRUMENTED_YIELD_VALUE 247 -#define INSTRUMENTED_END_ASYNC_FOR 248 -#define INSTRUMENTED_LOAD_SUPER_ATTR 249 -#define INSTRUMENTED_CALL 250 -#define INSTRUMENTED_CALL_KW 251 -#define INSTRUMENTED_CALL_FUNCTION_EX 252 -#define INSTRUMENTED_JUMP_BACKWARD 253 -#define INSTRUMENTED_LINE 254 -#define ENTER_EXECUTOR 255 +#define BINARY_OP_SUBSCR_USTR_INT 141 +#define BINARY_OP_SUBTRACT_FLOAT 142 +#define BINARY_OP_SUBTRACT_INT 143 +#define CALL_ALLOC_AND_ENTER_INIT 144 +#define CALL_BOUND_METHOD_EXACT_ARGS 145 +#define CALL_BOUND_METHOD_GENERAL 146 +#define CALL_BUILTIN_CLASS 147 +#define CALL_BUILTIN_FAST 148 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 149 +#define CALL_BUILTIN_O 150 +#define CALL_EX_NON_PY_GENERAL 151 +#define CALL_EX_PY 152 +#define CALL_ISINSTANCE 153 +#define CALL_KW_BOUND_METHOD 154 +#define CALL_KW_NON_PY 155 +#define CALL_KW_PY 156 +#define CALL_LEN 157 +#define CALL_LIST_APPEND 158 +#define CALL_METHOD_DESCRIPTOR_FAST 159 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 160 +#define CALL_METHOD_DESCRIPTOR_NOARGS 161 +#define CALL_METHOD_DESCRIPTOR_O 162 +#define CALL_NON_PY_GENERAL 163 +#define CALL_PY_EXACT_ARGS 164 +#define CALL_PY_GENERAL 165 +#define CALL_STR_1 166 +#define CALL_TUPLE_1 167 +#define CALL_TYPE_1 168 +#define COMPARE_OP_FLOAT 169 +#define COMPARE_OP_INT 170 +#define COMPARE_OP_STR 171 +#define CONTAINS_OP_DICT 172 +#define CONTAINS_OP_SET 173 +#define FOR_ITER_GEN 174 +#define FOR_ITER_LIST 175 +#define FOR_ITER_RANGE 176 +#define FOR_ITER_TUPLE 177 +#define FOR_ITER_VIRTUAL 178 +#define GET_ITER_SELF 179 +#define GET_ITER_VIRTUAL 180 +#define JUMP_BACKWARD_JIT 181 +#define JUMP_BACKWARD_NO_JIT 182 +#define LOAD_ATTR_CLASS 183 +#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 184 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 185 +#define LOAD_ATTR_INSTANCE_VALUE 186 +#define LOAD_ATTR_METHOD_LAZY_DICT 187 +#define LOAD_ATTR_METHOD_NO_DICT 188 +#define LOAD_ATTR_METHOD_WITH_VALUES 189 +#define LOAD_ATTR_MODULE 190 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 191 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 192 +#define LOAD_ATTR_PROPERTY 193 +#define LOAD_ATTR_SLOT 194 +#define LOAD_ATTR_WITH_HINT 195 +#define LOAD_GLOBAL_BUILTIN 196 +#define LOAD_GLOBAL_MODULE 197 +#define LOAD_SUPER_ATTR_ATTR 198 +#define LOAD_SUPER_ATTR_METHOD 199 +#define RESUME_CHECK 200 +#define RESUME_CHECK_JIT 201 +#define SEND_ASYNC_GEN 202 +#define SEND_GEN 203 +#define SEND_VIRTUAL 204 +#define STORE_ATTR_INSTANCE_VALUE 205 +#define STORE_ATTR_SLOT 206 +#define STORE_ATTR_WITH_HINT 207 +#define STORE_SUBSCR_DICT 208 +#define STORE_SUBSCR_LIST_INT 209 +#define TO_BOOL_ALWAYS_TRUE 210 +#define TO_BOOL_BOOL 211 +#define TO_BOOL_INT 212 +#define TO_BOOL_LIST 213 +#define TO_BOOL_NONE 214 +#define TO_BOOL_STR 215 +#define UNPACK_SEQUENCE_LIST 216 +#define UNPACK_SEQUENCE_TUPLE 217 +#define UNPACK_SEQUENCE_TWO_TUPLE 218 +#define INSTRUMENTED_END_FOR 233 +#define INSTRUMENTED_POP_ITER 234 +#define INSTRUMENTED_END_SEND 235 +#define INSTRUMENTED_FOR_ITER 236 +#define INSTRUMENTED_INSTRUCTION 237 +#define INSTRUMENTED_JUMP_FORWARD 238 +#define INSTRUMENTED_NOT_TAKEN 239 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 240 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 241 +#define INSTRUMENTED_POP_JUMP_IF_NONE 242 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 243 +#define INSTRUMENTED_RESUME 244 +#define INSTRUMENTED_RETURN_VALUE 245 +#define INSTRUMENTED_YIELD_VALUE 246 +#define INSTRUMENTED_END_ASYNC_FOR 247 +#define INSTRUMENTED_LOAD_SUPER_ATTR 248 +#define INSTRUMENTED_CALL 249 +#define INSTRUMENTED_CALL_KW 250 +#define INSTRUMENTED_CALL_FUNCTION_EX 251 +#define INSTRUMENTED_JUMP_BACKWARD 252 +#define INSTRUMENTED_LINE 253 +#define ENTER_EXECUTOR 254 +#define TRACE_RECORD 255 #define ANNOTATIONS_PLACEHOLDER 256 #define JUMP 257 #define JUMP_IF_FALSE 258 @@ -247,9 +256,9 @@ extern "C" { #define SETUP_WITH 265 #define STORE_FAST_MAYBE_NULL 266 -#define HAVE_ARGUMENT 43 +#define HAVE_ARGUMENT 41 #define MIN_SPECIALIZED_OPCODE 129 -#define MIN_INSTRUMENTED_OPCODE 234 +#define MIN_INSTRUMENTED_OPCODE 233 #ifdef __cplusplus } diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 532873b51e65bba..d82d52584069413 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -6,6 +6,9 @@ configure.ac must also be changed. There is also (independent) API version information in modsupport.h. + + This header should self-contained; PC/python_ver_rc.h includes it + without the rest of Python.h. */ /* Values for PY_RELEASE_LEVEL */ @@ -18,13 +21,13 @@ /* Version parsed out into numeric values */ /*--start constants--*/ #define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 15 +#define PY_MINOR_VERSION 16 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.15.0a0" +#define PY_VERSION "3.16.0a0" /*--end constants--*/ @@ -46,4 +49,16 @@ // Public Py_PACK_VERSION is declared in pymacro.h; it needs . + +/* The API and ABI versions are left for backwards compatibility. + They've not been updated since 2006 and 2010, respectively. + API/ABI versioning is now tied to the CPython version. + The *_VERSION and *_STRING symbols should define the same value; as + number and string literal respectively. Make sure the definitions match. +*/ +#define PYTHON_API_VERSION 1013 +#define PYTHON_API_STRING "1013" +#define PYTHON_ABI_VERSION 3 +#define PYTHON_ABI_STRING "3" + #endif //_Py_PATCHLEVEL_H diff --git a/Include/pyabi.h b/Include/pyabi.h new file mode 100644 index 000000000000000..21a6ab0c1ee6ea2 --- /dev/null +++ b/Include/pyabi.h @@ -0,0 +1,123 @@ +/* Macros that restrict available definitions and select implementations + * to match an ABI stability promise: + * + * - internal API/ABI (may change at any time) -- Py_BUILD_CORE* + * - general CPython API/ABI (may change in 3.x.0) -- default + * - Stable ABI: abi3, abi3t (long-term stable) -- Py_LIMITED_API, + * Py_TARGET_ABI3T, _Py_OPAQUE_PYOBJECT + * - Free-threading (incompatible with non-free-threading builds) + * -- Py_GIL_DISABLED + */ + +#ifndef _Py_PYABI_H +#define _Py_PYABI_H + +/* Defines to build Python and its standard library: + * + * - Py_BUILD_CORE: Build Python core. Gives access to Python internals; should + * not be used by third-party modules. + * - Py_BUILD_CORE_BUILTIN: Build a Python stdlib module as a built-in module. + * - Py_BUILD_CORE_MODULE: Build a Python stdlib module as a dynamic library. + * + * Py_BUILD_CORE_BUILTIN and Py_BUILD_CORE_MODULE imply Py_BUILD_CORE. + * + * On Windows, Py_BUILD_CORE_MODULE exports "PyInit_xxx" symbol, whereas + * Py_BUILD_CORE_BUILTIN does not. + */ +#if defined(Py_BUILD_CORE_BUILTIN) && !defined(Py_BUILD_CORE) +# define Py_BUILD_CORE +#endif +#if defined(Py_BUILD_CORE_MODULE) && !defined(Py_BUILD_CORE) +# define Py_BUILD_CORE +#endif + +/* Check valid values for target ABI macros. + */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 3 + // Empty Py_LIMITED_API used to work; redefine to + // Python 3.2 to be explicit. +# undef Py_LIMITED_API +# define Py_LIMITED_API 0x03020000 +#endif +#if defined(Py_TARGET_ABI3T) && Py_TARGET_ABI3T+0 < 0x030f0000 +# error "Py_TARGET_ABI3T must be 0x030f0000 (3.15) or above" +#endif + +/* Stable ABI for free-threaded builds (abi3t, introduced in PEP 803) + * is enabled by one of: + * - Py_TARGET_ABI3T, or + * - Py_LIMITED_API and Py_GIL_DISABLED. + * + * These affect set the following, which Python.h should use internally: + * - Py_LIMITED_API (defines the subset of API we expose) + * - _Py_OPAQUE_PYOBJECT (additionally hides what's ABI-incompatible between + * free-threaded & GIL) + * + * (Don't use Py_TARGET_ABI3T directly. It's currently only used to set these + * 2 macros, and defined for users' convenience.) + * + * This logic is currently partially duplicated in PC/pyconfig.h. + */ +#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) \ + && !defined(Py_TARGET_ABI3T) +# define Py_TARGET_ABI3T Py_LIMITED_API +#endif +#if defined(Py_TARGET_ABI3T) +# define _Py_OPAQUE_PYOBJECT +# if !defined(Py_LIMITED_API) +# define Py_LIMITED_API Py_TARGET_ABI3T +# elif Py_LIMITED_API > Py_TARGET_ABI3T + // if both are defined, use the *lower* version, + // i.e. maximum compatibility +# undef Py_LIMITED_API +# define Py_LIMITED_API Py_TARGET_ABI3T +# endif +#else +# ifdef _Py_OPAQUE_PYOBJECT + // _Py_OPAQUE_PYOBJECT is a private macro; do not define it directly. +# error "Define Py_TARGET_ABI3T to target abi3t." +# endif +#endif + +#if defined(Py_TARGET_ABI3T) +# if !defined(Py_GIL_DISABLED) + // Define Py_GIL_DISABLED for users' needs. Users check this macro to see + // whether they need extra synchronization. +# define Py_GIL_DISABLED +# endif +# if defined(_Py_IS_TESTCEXT) + // When compiling for abi3t, contents of Python.h should not depend + // on Py_GIL_DISABLED. + // We ask GCC to error if it sees the macro from this point on. + // Since users are free to the macro, and there's no way to undo the + // poisoning at the end of Python.h, we only do this in a test module + // (test_cext). + // + // Clang's poisoning is stricter than GCC's: it looks in `#elif` + // expressions after matching `#if`s. We disable it for now. + // We also provide an undocumented, unsupported opt-out macro to help + // porting to other compilers. Consider reaching out if you use it. +# if defined(__GNUC__) && !defined(__clang__) && !defined(_Py_NO_GCC_POISON) +# undef Py_GIL_DISABLED +# pragma GCC poison Py_GIL_DISABLED +# endif +# endif +#endif + +/* The internal C API must not be used with the limited C API: make sure + * that Py_BUILD_CORE* macros are not defined in this case. + * But, keep the "original" values, under different names, for "exports.h" + */ +#ifdef Py_BUILD_CORE +# define _PyEXPORTS_CORE +#endif +#ifdef Py_BUILD_CORE_MODULE +# define _PyEXPORTS_CORE_MODULE +#endif +#ifdef Py_LIMITED_API +# undef Py_BUILD_CORE +# undef Py_BUILD_CORE_BUILTIN +# undef Py_BUILD_CORE_MODULE +#endif + +#endif // _Py_PYABI_H diff --git a/Include/pybuffer.h b/Include/pybuffer.h index ca1c6058d9052c5..6371b9b483777ba 100644 --- a/Include/pybuffer.h +++ b/Include/pybuffer.h @@ -67,27 +67,27 @@ PyAPI_FUNC(int) PyBuffer_FromContiguous(const Py_buffer *view, const void *buf, error (i.e. the object does not have a buffer interface or it is not working). - If fort is 'F', then if the object is multi-dimensional, + If order is 'F', then if the object is multi-dimensional, then the data will be copied into the array in Fortran-style (first dimension varies the fastest). If - fort is 'C', then the data will be copied into the array - in C-style (last dimension varies the fastest). If fort + order is 'C', then the data will be copied into the array + in C-style (last dimension varies the fastest). If order is 'A', then it does not matter and the copy will be made in whatever way is more efficient. */ PyAPI_FUNC(int) PyObject_CopyData(PyObject *dest, PyObject *src); /* Copy the data from the src buffer to the buffer of destination. */ -PyAPI_FUNC(int) PyBuffer_IsContiguous(const Py_buffer *view, char fort); +PyAPI_FUNC(int) PyBuffer_IsContiguous(const Py_buffer *view, char order); /*Fill the strides array with byte-strides of a contiguous - (Fortran-style if fort is 'F' or C-style otherwise) + (Fortran-style if order is 'F' or C-style otherwise) array of the given shape with the given number of bytes per element. */ PyAPI_FUNC(void) PyBuffer_FillContiguousStrides(int ndims, Py_ssize_t *shape, Py_ssize_t *strides, int itemsize, - char fort); + char order); /* Fills in a buffer-info structure correctly for an exporter that can only share a contiguous chunk of memory of diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 5d0028c116e2d86..cfabbc5fe8d5da0 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -91,6 +91,9 @@ PyAPI_DATA(PyObject *) PyExc_EOFError; PyAPI_DATA(PyObject *) PyExc_FloatingPointError; PyAPI_DATA(PyObject *) PyExc_OSError; PyAPI_DATA(PyObject *) PyExc_ImportError; +#if !defined(Py_LIMITED_API) +PyAPI_DATA(PyObject *) PyExc_ImportCycleError; +#endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03060000 PyAPI_DATA(PyObject *) PyExc_ModuleNotFoundError; #endif diff --git a/Include/pyexpat.h b/Include/pyexpat.h index 9824d099c3df7d0..a676e16a7a457ea 100644 --- a/Include/pyexpat.h +++ b/Include/pyexpat.h @@ -52,6 +52,19 @@ struct PyExpat_CAPI int (*SetHashSalt)(XML_Parser parser, unsigned long hash_salt); /* might be NULL for expat < 2.6.0 */ XML_Bool (*SetReparseDeferralEnabled)(XML_Parser parser, XML_Bool enabled); + /* might be NULL for expat < 2.7.2 */ + XML_Bool (*SetAllocTrackerActivationThreshold)( + XML_Parser parser, unsigned long long activationThresholdBytes); + XML_Bool (*SetAllocTrackerMaximumAmplification)( + XML_Parser parser, float maxAmplificationFactor); + /* might be NULL for expat < 2.4.0 */ + XML_Bool (*SetBillionLaughsAttackProtectionActivationThreshold)( + XML_Parser parser, unsigned long long activationThresholdBytes); + XML_Bool (*SetBillionLaughsAttackProtectionMaximumAmplification)( + XML_Parser parser, float maxAmplificationFactor); + /* might be NULL for expat < 2.8.0 */ + XML_Bool (*SetHashSalt16Bytes)( + XML_Parser parser, const uint8_t entropy[16]); /* always add new stuff to the end! */ }; diff --git a/Include/pylock.h b/Include/pylock.h deleted file mode 100644 index 1939ef269d30906..000000000000000 --- a/Include/pylock.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef Py_LOCK_H -#define Py_LOCK_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_LOCK_H -# include "cpython/pylock.h" -# undef Py_CPYTHON_LOCK_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_LOCK_H */ diff --git a/Include/pymacconfig.h b/Include/pymacconfig.h index 615abe103ca0388..9d63ddf8a716f43 100644 --- a/Include/pymacconfig.h +++ b/Include/pymacconfig.h @@ -18,7 +18,6 @@ #undef SIZEOF_UINTPTR_T #undef SIZEOF_PTHREAD_T #undef WORDS_BIGENDIAN -#undef DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754 #undef DOUBLE_IS_BIG_ENDIAN_IEEE754 #undef DOUBLE_IS_LITTLE_ENDIAN_IEEE754 #undef HAVE_GCC_ASM_FOR_X87 diff --git a/Include/pymacro.h b/Include/pymacro.h index b2886ddac5d17ed..7ecce44a0d2a428 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -86,6 +86,28 @@ # endif #endif + +// _Py_ANONYMOUS: modifier for declaring an anonymous union. +// Usage: _Py_ANONYMOUS union { ... }; +// Standards/compiler support: +// - C++ allows anonymous unions, but not structs +// - C11 and above allows anonymous unions and structs +// - MSVC has warning(disable: 4201) "nonstandard extension used : nameless +// struct/union". This is specific enough that we disable it for all of +// Python.h. +// - GCC & clang needs __extension__ before C11 +// To allow unsupported platforms which need other spellings, we use a +// predefined value of _Py_ANONYMOUS if it exists. +#ifndef _Py_ANONYMOUS +# if (defined(__GNUC__) || defined(__clang__)) \ + && !(defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) +# define _Py_ANONYMOUS __extension__ +# else +# define _Py_ANONYMOUS +# endif +#endif + + /* Minimum value between x and y */ #define Py_MIN(x, y) (((x) > (y)) ? (y) : (x)) @@ -94,6 +116,12 @@ /* Absolute value of the number x */ #define Py_ABS(x) ((x) < 0 ? -(x) : (x)) +/* Safer implementation that avoids an undefined behavior for the minimal + value of the signed integer type if its absolute value is larger than + the maximal value of the signed integer type (in the two's complement + representations, which is common). + */ +#define _Py_ABS_CAST(T, x) ((x) >= 0 ? ((T) (x)) : ((T) (((T) -((x) + 1)) + 1u))) #define _Py_XSTRINGIFY(x) #x diff --git a/Include/pymath.h b/Include/pymath.h index 0ead1f95670fdee..7cfe441365df78d 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -7,6 +7,7 @@ /* High precision definition of pi and e (Euler) * The values are taken from libc6's math.h. */ +// Deprecated since Python 3.15. #ifndef Py_MATH_PIl #define Py_MATH_PIl 3.1415926535897932384626433832795029L #endif @@ -14,6 +15,7 @@ #define Py_MATH_PI 3.14159265358979323846 #endif +// Deprecated since Python 3.15. #ifndef Py_MATH_El #define Py_MATH_El 2.7182818284590452353602874713526625L #endif @@ -43,13 +45,14 @@ #define Py_IS_FINITE(X) isfinite(X) // Py_INFINITY: Value that evaluates to a positive double infinity. +// Soft deprecated since Python 3.15, use INFINITY instead. #ifndef Py_INFINITY # define Py_INFINITY ((double)INFINITY) #endif /* Py_HUGE_VAL should always be the same as Py_INFINITY. But historically * this was not reliable and Python did not require IEEE floats and C99 - * conformity. The macro was soft deprecated in Python 3.14, use Py_INFINITY instead. + * conformity. The macro was soft deprecated in Python 3.14, use INFINITY instead. */ #ifndef Py_HUGE_VAL # define Py_HUGE_VAL HUGE_VAL @@ -57,9 +60,24 @@ /* Py_NAN: Value that evaluates to a quiet Not-a-Number (NaN). The sign is * undefined and normally not relevant, but e.g. fixed for float("nan"). + * + * Note: On Solaris, NAN is a function address, hence arithmetic is impossible. + * For that reason, we instead use the built-in call if available or fallback + * to a generic NaN computed from strtod() as a last resort. + * + * See https://github.com/python/cpython/issues/136006 for details. */ #if !defined(Py_NAN) -# define Py_NAN ((double)NAN) +# if defined(__sun) +# if _Py__has_builtin(__builtin_nanf) +# define Py_NAN ((double)__builtin_nanf("")) +# else +# include +# define Py_NAN (strtod("NAN", NULL)) +# endif +# else +# define Py_NAN ((double)NAN) +# endif #endif #endif /* Py_PYMATH_H */ diff --git a/Include/pyport.h b/Include/pyport.h index 89829373be2ca2b..73a3e6cdaf09200 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -50,7 +50,7 @@ // to prevent C++ compiler warnings. On C23 and newer and on C++11 and newer, // _Py_NULL is defined as nullptr. #if !defined(_MSC_VER) && \ - ((defined (__STDC_VERSION__) && __STDC_VERSION__ > 201710L) \ + ((defined (__STDC_VERSION__) && __STDC_VERSION__ >= 202311L) \ || (defined(__cplusplus) && __cplusplus >= 201103)) # define _Py_NULL nullptr #else @@ -58,26 +58,6 @@ #endif -/* Defines to build Python and its standard library: - * - * - Py_BUILD_CORE: Build Python core. Give access to Python internals, but - * should not be used by third-party modules. - * - Py_BUILD_CORE_BUILTIN: Build a Python stdlib module as a built-in module. - * - Py_BUILD_CORE_MODULE: Build a Python stdlib module as a dynamic library. - * - * Py_BUILD_CORE_BUILTIN and Py_BUILD_CORE_MODULE imply Py_BUILD_CORE. - * - * On Windows, Py_BUILD_CORE_MODULE exports "PyInit_xxx" symbol, whereas - * Py_BUILD_CORE_BUILTIN does not. - */ -#if defined(Py_BUILD_CORE_BUILTIN) && !defined(Py_BUILD_CORE) -# define Py_BUILD_CORE -#endif -#if defined(Py_BUILD_CORE_MODULE) && !defined(Py_BUILD_CORE) -# define Py_BUILD_CORE -#endif - - /************************************************************************** Symbols and macros to supply platform-independent interfaces to basic C language & library operations whose spellings vary across platforms. @@ -385,17 +365,6 @@ extern "C" { # define Py_NO_INLINE #endif -#include "exports.h" - -#ifdef Py_LIMITED_API - // The internal C API must not be used with the limited C API: make sure - // that Py_BUILD_CORE macro is not defined in this case. These 3 macros are - // used by exports.h, so only undefine them afterwards. -# undef Py_BUILD_CORE -# undef Py_BUILD_CORE_BUILTIN -# undef Py_BUILD_CORE_MODULE -#endif - /* limits.h constants that may be missing */ #ifndef INT_MAX @@ -446,7 +415,9 @@ extern "C" { /* * Specify alignment on compilers that support it. */ -#if defined(__GNUC__) && __GNUC__ >= 3 +#ifdef Py_BUILD_CORE +// always use _Py_ALIGNED_DEF instead +#elif defined(__GNUC__) && __GNUC__ >= 3 #define Py_ALIGNED(x) __attribute__((aligned(x))) #else #define Py_ALIGNED(x) @@ -504,28 +475,30 @@ extern "C" { * Thread support is stubbed and any attempt to create a new thread fails. */ #if (!defined(HAVE_PTHREAD_STUBS) && \ + !defined(__wasi__) && \ (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))) # define Py_CAN_START_THREADS 1 #endif + +/* gh-142163: Some libraries rely on HAVE_THREAD_LOCAL being undefined, so + * we can only define it only when Py_BUILD_CORE is set.*/ +#ifdef Py_BUILD_CORE +// This is no longer coupled to _Py_thread_local. +# define HAVE_THREAD_LOCAL 1 +#endif + #ifdef WITH_THREAD -# ifdef Py_BUILD_CORE -# ifdef HAVE_THREAD_LOCAL -# error "HAVE_THREAD_LOCAL is already defined" -# endif -# define HAVE_THREAD_LOCAL 1 -# ifdef thread_local -# define _Py_thread_local thread_local -# elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__) -# define _Py_thread_local _Thread_local -# elif defined(_MSC_VER) /* AKA NT_THREADS */ -# define _Py_thread_local __declspec(thread) -# elif defined(__GNUC__) /* includes clang */ -# define _Py_thread_local __thread -# else - // fall back to the PyThread_tss_*() API, or ignore. -# undef HAVE_THREAD_LOCAL -# endif +# ifdef thread_local +# define _Py_thread_local thread_local +# elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__) +# define _Py_thread_local _Thread_local +# elif defined(_MSC_VER) /* AKA NT_THREADS */ +# define _Py_thread_local __declspec(thread) +# elif defined(__GNUC__) /* includes clang */ +# define _Py_thread_local __thread +# else +# error "no supported thread-local variable storage classifier" # endif #endif @@ -565,8 +538,11 @@ extern "C" { // // Example: _Py_TYPEOF(x) x_copy = (x); // -// The macro is only defined if GCC or clang compiler is used. -#if defined(__GNUC__) || defined(__clang__) +// On C23, use typeof(). Otherwise, the macro is only defined +// if GCC or clang compiler is used. +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +# define _Py_TYPEOF(expr) typeof(expr) +#elif defined(__GNUC__) || defined(__clang__) # define _Py_TYPEOF(expr) __typeof__(expr) #endif @@ -577,6 +553,7 @@ extern "C" { # if !defined(_Py_MEMORY_SANITIZER) # define _Py_MEMORY_SANITIZER # define _Py_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory)) +# define _Py_MSAN_UNPOISON(PTR, SIZE) (__msan_unpoison(PTR, SIZE)) # endif # endif # if __has_feature(address_sanitizer) @@ -615,6 +592,9 @@ extern "C" { #ifndef _Py_NO_SANITIZE_MEMORY # define _Py_NO_SANITIZE_MEMORY #endif +#ifndef _Py_MSAN_UNPOISON +# define _Py_MSAN_UNPOISON(PTR, SIZE) +#endif /* AIX has __bool__ redefined in it's system header file. */ #if defined(_AIX) && defined(__bool__) @@ -682,4 +662,10 @@ extern "C" { #endif +// Assume the stack grows down unless specified otherwise +#ifndef _Py_STACK_GROWS_DOWN +# define _Py_STACK_GROWS_DOWN 1 +#endif + + #endif /* Py_PYPORT_H */ diff --git a/Include/pystate.h b/Include/pystate.h index 727b8fbfffe0e67..8dad748238f4f39 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -120,6 +120,29 @@ PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE); PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); +/* PEP 788 -- Protection against interpreter finalization */ + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) + +typedef struct PyInterpreterGuard PyInterpreterGuard; +typedef struct PyInterpreterView PyInterpreterView; + +typedef void PyThreadStateToken; + +PyAPI_FUNC(PyInterpreterGuard *) PyInterpreterGuard_FromCurrent(void); +PyAPI_FUNC(void) PyInterpreterGuard_Close(PyInterpreterGuard *guard); +PyAPI_FUNC(PyInterpreterGuard *) PyInterpreterGuard_FromView(PyInterpreterView *view); + +PyAPI_FUNC(PyInterpreterView *) PyInterpreterView_FromCurrent(void); +PyAPI_FUNC(void) PyInterpreterView_Close(PyInterpreterView *view); +PyAPI_FUNC(PyInterpreterView *) PyInterpreterView_FromMain(void); + +PyAPI_FUNC(PyThreadStateToken *) PyThreadState_Ensure(PyInterpreterGuard *guard); +PyAPI_FUNC(PyThreadStateToken *) PyThreadState_EnsureFromView(PyInterpreterView *view); +PyAPI_FUNC(void) PyThreadState_Release(PyThreadStateToken *tstate); + +#endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_PYSTATE_H # include "cpython/pystate.h" diff --git a/Include/pythread.h b/Include/pythread.h index 82247daf8e0aa09..a8a28b8572acb6e 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -42,7 +42,8 @@ PyAPI_FUNC(unsigned long) PyThread_get_thread_ident(void); #if (defined(__APPLE__) || defined(__linux__) || defined(_WIN32) \ || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \ || defined(__OpenBSD__) || defined(__NetBSD__) \ - || defined(__DragonFly__) || defined(_AIX)) + || defined(__DragonFly__) || defined(_AIX) \ + || (defined(__sun__) && SIZEOF_LONG >= 8)) #define PY_HAVE_THREAD_NATIVE_ID PyAPI_FUNC(unsigned long) PyThread_get_thread_native_id(void); #endif diff --git a/Include/pytypedefs.h b/Include/pytypedefs.h index e78ed56a3b67cd1..1d0b4e3e85ef74a 100644 --- a/Include/pytypedefs.h +++ b/Include/pytypedefs.h @@ -14,6 +14,7 @@ typedef struct PyModuleDef_Slot PyModuleDef_Slot; typedef struct PyMethodDef PyMethodDef; typedef struct PyGetSetDef PyGetSetDef; typedef struct PyMemberDef PyMemberDef; +typedef struct PySlot PySlot; typedef struct _object PyObject; typedef struct _longobject PyLongObject; diff --git a/Include/refcount.h b/Include/refcount.h index ba34461fefcbb08..80fe7ff70a11e87 100644 --- a/Include/refcount.h +++ b/Include/refcount.h @@ -5,6 +5,7 @@ extern "C" { #endif +#if !defined(_Py_OPAQUE_PYOBJECT) /* Immortalization: @@ -90,14 +91,16 @@ check by comparing the reference count field to the minimum immortality refcount # define _Py_REF_SHARED(refcnt, flags) \ (((refcnt) << _Py_REF_SHARED_SHIFT) + (flags)) #endif // Py_GIL_DISABLED - +#endif // _Py_OPAQUE_PYOBJECT // Py_REFCNT() implementation for the stable ABI PyAPI_FUNC(Py_ssize_t) Py_REFCNT(PyObject *ob); #if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030e0000 // Stable ABI implements Py_REFCNT() as a function call - // on limited C API version 3.14 and newer. + // on limited C API version 3.14 and newer, and on abi3t. +#elif defined(_Py_OPAQUE_PYOBJECT) + // Py_REFCNT() is also a function call in abi3t. #else static inline Py_ssize_t _Py_REFCNT(PyObject *ob) { #if !defined(Py_GIL_DISABLED) @@ -114,6 +117,8 @@ PyAPI_FUNC(Py_ssize_t) Py_REFCNT(PyObject *ob); } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_REFCNT(ob) _Py_REFCNT(_PyObject_CAST(ob)) + #else + # define Py_REFCNT(ob) _Py_REFCNT(ob) #endif #endif @@ -124,7 +129,7 @@ static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) return (_Py_atomic_load_uint32_relaxed(&op->ob_ref_local) == _Py_IMMORTAL_REFCNT_LOCAL); #elif SIZEOF_VOID_P > 4 - return _Py_CAST(PY_INT32_T, op->ob_refcnt) < 0; + return _Py_CAST(int32_t, op->ob_refcnt) < 0; #else return op->ob_refcnt >= _Py_IMMORTAL_MINIMUM_REFCNT; #endif @@ -148,9 +153,10 @@ PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt); static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { assert(refcnt >= 0); -#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030d0000 +#if (defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030d0000) \ + || defined(_Py_OPAQUE_PYOBJECT) // Stable ABI implements Py_SET_REFCNT() as a function call - // on limited C API version 3.13 and newer. + // on limited C API version 3.13 and newer, and abi3t. _Py_SetRefcnt(ob, refcnt); #else // This immortal check is for code that is unaware of immortal objects. @@ -162,7 +168,7 @@ static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { } #ifndef Py_GIL_DISABLED #if SIZEOF_VOID_P > 4 - ob->ob_refcnt = (PY_UINT32_T)refcnt; + ob->ob_refcnt = (uint32_t)refcnt; #else ob->ob_refcnt = refcnt; #endif @@ -189,7 +195,7 @@ static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { ob->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); } #endif // Py_GIL_DISABLED -#endif // Py_LIMITED_API+0 < 0x030d0000 +#endif // Py_LIMITED_API } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SET_REFCNT(ob, refcnt) Py_SET_REFCNT(_PyObject_CAST(ob), (refcnt)) @@ -248,10 +254,11 @@ PyAPI_FUNC(void) _Py_DecRef(PyObject *); static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) { -#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) +#if (defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))) \ + || defined(_Py_OPAQUE_PYOBJECT) // Stable ABI implements Py_INCREF() as a function call on limited C API - // version 3.12 and newer, and on Python built in debug mode. _Py_IncRef() - // was added to Python 3.10.0a7, use Py_IncRef() on older Python versions. + // version 3.12 and newer, abi3t, and on Python built in debug mode. + // _Py_IncRef() was added to Python 3.10.0a7, use Py_IncRef() on older versions. // Py_IncRef() accepts NULL whereas _Py_IncRef() doesn't. # if Py_LIMITED_API+0 >= 0x030a00A7 _Py_IncRef(op); @@ -276,7 +283,7 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) _Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT)); } #elif SIZEOF_VOID_P > 4 - PY_UINT32_T cur_refcnt = op->ob_refcnt; + uint32_t cur_refcnt = op->ob_refcnt; if (cur_refcnt >= _Py_IMMORTAL_INITIAL_REFCNT) { // the object is immortal _Py_INCREF_IMMORTAL_STAT_INC(); @@ -303,8 +310,8 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) # define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op)) #endif - -#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +#if !defined(Py_LIMITED_API) +#if defined(Py_GIL_DISABLED) // Implements Py_DECREF on objects not owned by the current thread. PyAPI_FUNC(void) _Py_DecRefShared(PyObject *); PyAPI_FUNC(void) _Py_DecRefSharedDebug(PyObject *, const char *, int); @@ -314,12 +321,14 @@ PyAPI_FUNC(void) _Py_DecRefSharedDebug(PyObject *, const char *, int); // zero. Otherwise, the thread gives up ownership and merges the reference // count fields. PyAPI_FUNC(void) _Py_MergeZeroLocalRefcount(PyObject *); -#endif +#endif // Py_GIL_DISABLED +#endif // Py_LIMITED_API -#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) +#if (defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))) \ + || defined(_Py_OPAQUE_PYOBJECT) // Stable ABI implements Py_DECREF() as a function call on limited C API -// version 3.12 and newer, and on Python built in debug mode. _Py_DecRef() was -// added to Python 3.10.0a7, use Py_DecRef() on older Python versions. +// version 3.12 and newer, abi3t, and on Python built in debug mode. +// _Py_DecRef() was added to Python 3.10.0a7, use Py_DecRef() on older versions. // Py_DecRef() accepts NULL whereas _Py_DecRef() doesn't. static inline void Py_DECREF(PyObject *op) { # if Py_LIMITED_API+0 >= 0x030a00A7 @@ -385,7 +394,7 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) #if SIZEOF_VOID_P > 4 /* If an object has been freed, it will have a negative full refcnt * If it has not it been freed, will have a very large refcnt */ - if (op->ob_refcnt_full <= 0 || op->ob_refcnt > (((PY_UINT32_T)-1) - (1<<20))) { + if (op->ob_refcnt_full <= 0 || op->ob_refcnt > (((uint32_t)-1) - (1<<20))) { #else if (op->ob_refcnt <= 0) { #endif diff --git a/Include/sliceobject.h b/Include/sliceobject.h index 35e2ea254ca80a4..9d6a16da95fe2f5 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -16,19 +16,6 @@ PyAPI_DATA(PyObject) _Py_EllipsisObject; /* Don't use this directly */ /* Slice object interface */ -/* - -A slice object containing start, stop, and step data members (the -names are from range). After much talk with Guido, it was decided to -let these be any arbitrary python type. Py_None stands for omitted values. -*/ -#ifndef Py_LIMITED_API -typedef struct { - PyObject_HEAD - PyObject *start, *stop, *step; /* not NULL */ -} PySliceObject; -#endif - PyAPI_DATA(PyTypeObject) PySlice_Type; PyAPI_DATA(PyTypeObject) PyEllipsis_Type; @@ -36,12 +23,6 @@ PyAPI_DATA(PyTypeObject) PyEllipsis_Type; PyAPI_FUNC(PyObject *) PySlice_New(PyObject* start, PyObject* stop, PyObject* step); -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PySlice_FromIndices(Py_ssize_t start, Py_ssize_t stop); -PyAPI_FUNC(int) _PySlice_GetLongIndices(PySliceObject *self, PyObject *length, - PyObject **start_ptr, PyObject **stop_ptr, - PyObject **step_ptr); -#endif PyAPI_FUNC(int) PySlice_GetIndices(PyObject *r, Py_ssize_t length, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step); Py_DEPRECATED(3.7) @@ -63,6 +44,12 @@ PyAPI_FUNC(Py_ssize_t) PySlice_AdjustIndices(Py_ssize_t length, Py_ssize_t step); #endif +#ifndef Py_LIMITED_API +# define _Py_CPYTHON_SLICEOBJECT_H +# include "cpython/sliceobject.h" +# undef _Py_CPYTHON_SLICEOBJECT_H +#endif + #ifdef __cplusplus } #endif diff --git a/Include/slots.h b/Include/slots.h new file mode 100644 index 000000000000000..4bf7bda0208a8df --- /dev/null +++ b/Include/slots.h @@ -0,0 +1,56 @@ +#ifndef _Py_HAVE_SLOTS_H +#define _Py_HAVE_SLOTS_H + +typedef void (*_Py_funcptr_t)(void); + +struct PySlot { + uint16_t sl_id; + uint16_t sl_flags; + _Py_ANONYMOUS union { + uint32_t sl_reserved; // must be 0 + }; + _Py_ANONYMOUS union { + void *sl_ptr; + _Py_funcptr_t sl_func; + Py_ssize_t sl_size; + int64_t sl_int64; + uint64_t sl_uint64; + }; +}; + +#define PySlot_OPTIONAL 0x0001 +#define PySlot_STATIC 0x0002 +#define PySlot_INTPTR 0x0004 + +#define Py_slot_invalid 0xffff + +#define PySlot_DATA(NAME, VALUE) \ + {.sl_id=(NAME), .sl_flags=PySlot_INTPTR, .sl_ptr=(void*)(VALUE)} + +#define PySlot_FUNC(NAME, VALUE) \ + {.sl_id=(NAME), .sl_func=(_Py_funcptr_t)(VALUE)} + +#define PySlot_SIZE(NAME, VALUE) \ + {.sl_id=(NAME), .sl_size=(Py_ssize_t)(VALUE)} + +#define PySlot_INT64(NAME, VALUE) \ + {.sl_id=(NAME), .sl_int64=(int64_t)(VALUE)} + +#define PySlot_UINT64(NAME, VALUE) \ + {.sl_id=(NAME), .sl_uint64=(uint64_t)(VALUE)} + +#define PySlot_STATIC_DATA(NAME, VALUE) \ + {.sl_id=(NAME), .sl_flags=PySlot_STATIC, .sl_ptr=(VALUE)} + +#define PySlot_END {0} + + +// Macros without designated initializers (for C++11 and below): + +#define PySlot_PTR(NAME, VALUE) \ + {(NAME), PySlot_INTPTR, {0}, {(void*)(VALUE)}} + +#define PySlot_PTR_STATIC(NAME, VALUE) \ + {(NAME), PySlot_INTPTR | PySlot_STATIC, {0}, {(void*)(VALUE)}} + +#endif // _Py_HAVE_SLOTS_H diff --git a/Include/slots_generated.h b/Include/slots_generated.h new file mode 100644 index 000000000000000..42edd3ad4c69ffd --- /dev/null +++ b/Include/slots_generated.h @@ -0,0 +1,121 @@ +/* Generated by Tools/build/generate_slots.py */ + +#ifndef _PY_HAVE_SLOTS_GENERATED_H +#define _PY_HAVE_SLOTS_GENERATED_H + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +#define _Py_SLOT_COMPAT_VALUE(OLD, NEW) NEW +#else +#define _Py_SLOT_COMPAT_VALUE(OLD, NEW) OLD +#endif + +#define Py_slot_end 0 +#define Py_mp_subscript 5 +#define Py_nb_absolute 6 +#define Py_nb_add 7 +#define Py_nb_and 8 +#define Py_nb_bool 9 +#define Py_nb_divmod 10 +#define Py_nb_float 11 +#define Py_nb_floor_divide 12 +#define Py_nb_index 13 +#define Py_nb_inplace_add 14 +#define Py_nb_inplace_and 15 +#define Py_nb_inplace_floor_divide 16 +#define Py_nb_inplace_lshift 17 +#define Py_nb_inplace_multiply 18 +#define Py_nb_inplace_or 19 +#define Py_nb_inplace_power 20 +#define Py_nb_inplace_remainder 21 +#define Py_nb_inplace_rshift 22 +#define Py_nb_inplace_subtract 23 +#define Py_nb_inplace_true_divide 24 +#define Py_nb_inplace_xor 25 +#define Py_nb_int 26 +#define Py_nb_invert 27 +#define Py_nb_lshift 28 +#define Py_nb_multiply 29 +#define Py_nb_negative 30 +#define Py_nb_or 31 +#define Py_nb_positive 32 +#define Py_nb_power 33 +#define Py_nb_remainder 34 +#define Py_nb_rshift 35 +#define Py_nb_subtract 36 +#define Py_nb_true_divide 37 +#define Py_nb_xor 38 +#define Py_sq_ass_item 39 +#define Py_sq_concat 40 +#define Py_sq_contains 41 +#define Py_sq_inplace_concat 42 +#define Py_sq_inplace_repeat 43 +#define Py_sq_item 44 +#define Py_sq_length 45 +#define Py_sq_repeat 46 +#define Py_tp_alloc 47 +#define Py_tp_base 48 +#define Py_tp_bases 49 +#define Py_tp_call 50 +#define Py_tp_clear 51 +#define Py_tp_dealloc 52 +#define Py_tp_del 53 +#define Py_tp_descr_get 54 +#define Py_tp_descr_set 55 +#define Py_tp_doc 56 +#define Py_tp_getattr 57 +#define Py_tp_getattro 58 +#define Py_tp_hash 59 +#define Py_tp_init 60 +#define Py_tp_is_gc 61 +#define Py_tp_iter 62 +#define Py_tp_iternext 63 +#define Py_tp_methods 64 +#define Py_tp_new 65 +#define Py_tp_repr 66 +#define Py_tp_richcompare 67 +#define Py_tp_setattr 68 +#define Py_tp_setattro 69 +#define Py_tp_str 70 +#define Py_tp_traverse 71 +#define Py_tp_members 72 +#define Py_tp_getset 73 +#define Py_tp_free 74 +#define Py_nb_matrix_multiply 75 +#define Py_nb_inplace_matrix_multiply 76 +#define Py_am_await 77 +#define Py_am_aiter 78 +#define Py_am_anext 79 +#define Py_tp_finalize 80 +#define Py_am_send 81 +#define Py_tp_vectorcall 82 +#define Py_tp_token 83 +#define Py_mod_create _Py_SLOT_COMPAT_VALUE(1, 84) +#define Py_mod_exec _Py_SLOT_COMPAT_VALUE(2, 85) +#define Py_mod_multiple_interpreters _Py_SLOT_COMPAT_VALUE(3, 86) +#define Py_mod_gil _Py_SLOT_COMPAT_VALUE(4, 87) +#define Py_bf_getbuffer _Py_SLOT_COMPAT_VALUE(1, 88) +#define Py_bf_releasebuffer _Py_SLOT_COMPAT_VALUE(2, 89) +#define Py_mp_ass_subscript _Py_SLOT_COMPAT_VALUE(3, 90) +#define Py_mp_length _Py_SLOT_COMPAT_VALUE(4, 91) +#define Py_slot_subslots 92 +#define Py_tp_slots 93 +#define Py_mod_slots 94 +#define Py_tp_name 95 +#define Py_tp_basicsize 96 +#define Py_tp_extra_basicsize 97 +#define Py_tp_itemsize 98 +#define Py_tp_flags 99 +#define Py_mod_name 100 +#define Py_mod_doc 101 +#define Py_mod_state_size 102 +#define Py_mod_methods 103 +#define Py_mod_state_traverse 104 +#define Py_mod_state_clear 105 +#define Py_mod_state_free 106 +#define Py_tp_metaclass 107 +#define Py_tp_module 108 +#define Py_mod_abi 109 +#define Py_mod_token 110 + +#define _Py_slot_COUNT 111 +#endif /* _PY_HAVE_SLOTS_GENERATED_H */ diff --git a/Include/structseq.h b/Include/structseq.h index 29e24fee54e6135..e5da785f13d46b8 100644 --- a/Include/structseq.h +++ b/Include/structseq.h @@ -21,12 +21,6 @@ typedef struct PyStructSequence_Desc { PyAPI_DATA(const char * const) PyStructSequence_UnnamedField; -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type, - PyStructSequence_Desc *desc); -PyAPI_FUNC(int) PyStructSequence_InitType2(PyTypeObject *type, - PyStructSequence_Desc *desc); -#endif PyAPI_FUNC(PyTypeObject*) PyStructSequence_NewType(PyStructSequence_Desc *desc); PyAPI_FUNC(PyObject *) PyStructSequence_New(PyTypeObject* type); @@ -35,9 +29,9 @@ PyAPI_FUNC(void) PyStructSequence_SetItem(PyObject*, Py_ssize_t, PyObject*); PyAPI_FUNC(PyObject*) PyStructSequence_GetItem(PyObject*, Py_ssize_t); #ifndef Py_LIMITED_API -typedef PyTupleObject PyStructSequence; -#define PyStructSequence_SET_ITEM PyStructSequence_SetItem -#define PyStructSequence_GET_ITEM PyStructSequence_GetItem +# define _Py_CPYTHON_STRUCTSEQ_H +# include "cpython/structseq.h" +# undef _Py_CPYTHON_STRUCTSEQ_H #endif #ifdef __cplusplus diff --git a/Include/sysmodule.h b/Include/sysmodule.h index 2f362791797ded6..b7d800c5e5db0b9 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -23,8 +23,6 @@ PyAPI_FUNC(void) PySys_WriteStderr(const char *format, ...) PyAPI_FUNC(void) PySys_FormatStdout(const char *format, ...); PyAPI_FUNC(void) PySys_FormatStderr(const char *format, ...); -Py_DEPRECATED(3.13) PyAPI_FUNC(void) PySys_ResetWarnOptions(void); - PyAPI_FUNC(PyObject *) PySys_GetXOptions(void); #ifdef __cplusplus diff --git a/Include/typeslots.h b/Include/typeslots.h deleted file mode 100644 index a7f3017ec02e92f..000000000000000 --- a/Include/typeslots.h +++ /dev/null @@ -1,96 +0,0 @@ -/* Do not renumber the file; these numbers are part of the stable ABI. */ -#define Py_bf_getbuffer 1 -#define Py_bf_releasebuffer 2 -#define Py_mp_ass_subscript 3 -#define Py_mp_length 4 -#define Py_mp_subscript 5 -#define Py_nb_absolute 6 -#define Py_nb_add 7 -#define Py_nb_and 8 -#define Py_nb_bool 9 -#define Py_nb_divmod 10 -#define Py_nb_float 11 -#define Py_nb_floor_divide 12 -#define Py_nb_index 13 -#define Py_nb_inplace_add 14 -#define Py_nb_inplace_and 15 -#define Py_nb_inplace_floor_divide 16 -#define Py_nb_inplace_lshift 17 -#define Py_nb_inplace_multiply 18 -#define Py_nb_inplace_or 19 -#define Py_nb_inplace_power 20 -#define Py_nb_inplace_remainder 21 -#define Py_nb_inplace_rshift 22 -#define Py_nb_inplace_subtract 23 -#define Py_nb_inplace_true_divide 24 -#define Py_nb_inplace_xor 25 -#define Py_nb_int 26 -#define Py_nb_invert 27 -#define Py_nb_lshift 28 -#define Py_nb_multiply 29 -#define Py_nb_negative 30 -#define Py_nb_or 31 -#define Py_nb_positive 32 -#define Py_nb_power 33 -#define Py_nb_remainder 34 -#define Py_nb_rshift 35 -#define Py_nb_subtract 36 -#define Py_nb_true_divide 37 -#define Py_nb_xor 38 -#define Py_sq_ass_item 39 -#define Py_sq_concat 40 -#define Py_sq_contains 41 -#define Py_sq_inplace_concat 42 -#define Py_sq_inplace_repeat 43 -#define Py_sq_item 44 -#define Py_sq_length 45 -#define Py_sq_repeat 46 -#define Py_tp_alloc 47 -#define Py_tp_base 48 -#define Py_tp_bases 49 -#define Py_tp_call 50 -#define Py_tp_clear 51 -#define Py_tp_dealloc 52 -#define Py_tp_del 53 -#define Py_tp_descr_get 54 -#define Py_tp_descr_set 55 -#define Py_tp_doc 56 -#define Py_tp_getattr 57 -#define Py_tp_getattro 58 -#define Py_tp_hash 59 -#define Py_tp_init 60 -#define Py_tp_is_gc 61 -#define Py_tp_iter 62 -#define Py_tp_iternext 63 -#define Py_tp_methods 64 -#define Py_tp_new 65 -#define Py_tp_repr 66 -#define Py_tp_richcompare 67 -#define Py_tp_setattr 68 -#define Py_tp_setattro 69 -#define Py_tp_str 70 -#define Py_tp_traverse 71 -#define Py_tp_members 72 -#define Py_tp_getset 73 -#define Py_tp_free 74 -#define Py_nb_matrix_multiply 75 -#define Py_nb_inplace_matrix_multiply 76 -#define Py_am_await 77 -#define Py_am_aiter 78 -#define Py_am_anext 79 -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000 -/* New in 3.5 */ -#define Py_tp_finalize 80 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 -/* New in 3.10 */ -#define Py_am_send 81 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 -/* New in 3.14 */ -#define Py_tp_vectorcall 82 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 -/* New in 3.14 */ -#define Py_tp_token 83 -#endif diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index b72d581ec25804b..29f1d1b01c161f4 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -64,13 +64,9 @@ Copyright (c) Corporation for National Research Initiatives. #error Must define SIZEOF_WCHAR_T #endif +/* Soft-deprecated defines */ #define Py_UNICODE_SIZE SIZEOF_WCHAR_T - -/* If wchar_t can be used for UCS-4 storage, set Py_UNICODE_WIDE. - Otherwise, Unicode strings are stored as UCS-2 (with limited support - for UTF-16) */ - -#if Py_UNICODE_SIZE >= 4 +#if SIZEOF_WCHAR_T >= 4 #define Py_UNICODE_WIDE #endif diff --git a/Include/weakrefobject.h b/Include/weakrefobject.h index a6e71eb178b124b..17fac62961c0fb8 100644 --- a/Include/weakrefobject.h +++ b/Include/weakrefobject.h @@ -27,7 +27,6 @@ PyAPI_FUNC(PyObject *) PyWeakref_NewRef(PyObject *ob, PyObject *callback); PyAPI_FUNC(PyObject *) PyWeakref_NewProxy(PyObject *ob, PyObject *callback); -Py_DEPRECATED(3.13) PyAPI_FUNC(PyObject *) PyWeakref_GetObject(PyObject *ref); #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030D0000 PyAPI_FUNC(int) PyWeakref_GetRef(PyObject *ref, PyObject **pobj); diff --git a/InternalDocs/README.md b/InternalDocs/README.md index 6b1d919826402c7..3e8ab4423157534 100644 --- a/InternalDocs/README.md +++ b/InternalDocs/README.md @@ -11,6 +11,11 @@ it is not, please report that through the [issue tracker](https://github.com/python/cpython/issues). +General Resources +--- + +- [Source Code Structure](structure.md) + Compiling Python Source Code --- @@ -29,13 +34,13 @@ Runtime Objects - [Frames](frames.md) -- [String Interning](string_interning.md) - Program Execution --- - [The Bytecode Interpreter](interpreter.md) +- [Stack references (_PyStackRef)](stackrefs.md) + - [The JIT](jit.md) - [Garbage Collector Design](garbage_collector.md) @@ -44,6 +49,16 @@ Program Execution - [Quiescent-State Based Reclamation (QSBR)](qsbr.md) +- [Stack protection](stack_protection.md) + +Built-in Types +--- + +- [String Interning](string_interning.md) + +- [List sort algorithm](../Objects/listsort.txt) + + Modules --- diff --git a/InternalDocs/asyncio.md b/InternalDocs/asyncio.md index 22159852ca54db6..ca7e8a92dec1ce5 100644 --- a/InternalDocs/asyncio.md +++ b/InternalDocs/asyncio.md @@ -205,7 +205,7 @@ the current task is found and returned. If no matching thread state is found, `None` is returned. In free-threading, it avoids contention on a global dictionary as -threads can access the current task of thier running loop without any +threads can access the current task of their running loop without any locking. --- diff --git a/InternalDocs/code_objects.md b/InternalDocs/code_objects.md index a91a7043c1b8d41..cccbe71588622cf 100644 --- a/InternalDocs/code_objects.md +++ b/InternalDocs/code_objects.md @@ -70,14 +70,6 @@ The `co_linetable` bytes object of code objects contains a compact representation of the source code positions of instructions, which are returned by the `co_positions()` iterator. -> [!NOTE] -> `co_linetable` is not to be confused with `co_lnotab`. -> For backwards compatibility, `co_lnotab` exposes the format -> as it existed in Python 3.10 and lower: this older format -> stores only the start line for each instruction. -> It is lazily created from `co_linetable` when accessed. -> See [`Objects/lnotab_notes.txt`](../Objects/lnotab_notes.txt) for more details. - `co_linetable` consists of a sequence of location entries. Each entry starts with a byte with the most significant bit set, followed by zero or more bytes with the most significant bit unset. diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md index 02bbdf6071fa12e..9ed4d0eb65a0bdd 100644 --- a/InternalDocs/compiler.md +++ b/InternalDocs/compiler.md @@ -359,19 +359,19 @@ in [Python/compile.c](../Python/compile.c) into a sequence of pseudo instruction These are similar to bytecode, but in some cases they are more abstract, and are resolved later into actual bytecode. The construction of this instruction sequence is handled by several functions that break the task down by various AST node types. -The functions are all named `compiler_visit_{xx}` where *xx* is the name of the node +The functions are all named `codegen_visit_{xx}` where *xx* is the name of the node type (such as `stmt`, `expr`, etc.). Each function receives a `struct compiler *` and `{xx}_ty` where *xx* is the AST node type. Typically these functions consist of a large 'switch' statement, branching based on the kind of node type passed to it. Simple things are handled inline in the 'switch' statement with more complex transformations farmed out to other -functions named `compiler_{xx}` with *xx* being a descriptive name of what is +functions named `codegen_{xx}` with *xx* being a descriptive name of what is being handled. When transforming an arbitrary AST node, use the `VISIT()` macro. -The appropriate `compiler_visit_{xx}` function is called, based on the value +The appropriate `codegen_visit_{xx}` function is called, based on the value passed in for (so `VISIT({c}, expr, {node})` calls -`compiler_visit_expr({c}, {node})`). The `VISIT_SEQ()` macro is very similar, +`codegen_visit_expr({c}, {node})`). The `VISIT_SEQ()` macro is very similar, but is called on AST node sequences (those values that were created as arguments to a node that used the '*' modifier). @@ -414,8 +414,8 @@ which is added at the end of a function is not associated with any line in the source code. There are several helper functions that will emit pseudo-instructions -and are named `compiler_{xx}()` where *xx* is what the function helps -with (`list`, `boolop`, etc.). A rather useful one is `compiler_nameop()`. +and are named `codegen_{xx}()` where *xx* is what the function helps +with (`list`, `boolop`, etc.). A rather useful one is `codegen_nameop()`. This function looks up the scope of a variable and, based on the expression context, emits the proper opcode to load, store, or delete the variable. @@ -604,4 +604,4 @@ References pp. 213--227, 1997. [^2]: The Zephyr Abstract Syntax Description Language.: - https://www.cs.princeton.edu/research/techreps/TR-554-97 + https://www.cs.princeton.edu/research/techreps/254 diff --git a/InternalDocs/frames.md b/InternalDocs/frames.md index 804d7436018a10e..60ab2055afa7b1c 100644 --- a/InternalDocs/frames.md +++ b/InternalDocs/frames.md @@ -111,6 +111,55 @@ The shim frame points to a special code object containing the `INTERPRETER_EXIT` instruction which cleans up the shim frame and returns. +### Base frame + +Each thread state contains an embedded `_PyInterpreterFrame` called the "base frame" +that serves as a sentinel at the bottom of the frame stack. This frame is allocated +in `_PyThreadStateImpl` (the internal extension of `PyThreadState`) and initialized +when the thread state is created. The `owner` field is set to `FRAME_OWNED_BY_INTERPRETER`. + +External profilers and sampling tools can validate that they have successfully unwound +the complete call stack by checking that the frame chain terminates at the base frame. +The `PyThreadState.base_frame` pointer provides the expected address to compare against. +If a stack walk doesn't reach this frame, the sample is incomplete (possibly due to a +race condition) and should be discarded. + +The base frame is embedded in `_PyThreadStateImpl` rather than `PyThreadState` because +`_PyInterpreterFrame` is defined in internal headers that cannot be exposed in the +public API. A pointer (`PyThreadState.base_frame`) is provided for profilers to access +the address without needing internal headers. + +See the initialization in `new_threadstate()` in [Python/pystate.c](../Python/pystate.c). + +#### How profilers should use the base frame + +External profilers should read `tstate->base_frame` before walking the stack, then +walk from `tstate->current_frame` following `frame->previous` pointers until reaching +a frame with `owner == FRAME_OWNED_BY_INTERPRETER`. After the walk, verify that the +last frame address matches `base_frame`. If not, discard the sample as incomplete +since the frame chain may have been in an inconsistent state due to concurrent updates. + + +### Remote Profiling Frame Cache + +The `last_profiled_frame` field in `PyThreadState` supports an optimization for +remote profilers that sample call stacks from external processes. When a remote +profiler reads the call stack, it writes the current frame address to this field. +The eval loop then keeps this pointer valid by updating it to the parent frame +whenever a frame returns (in `_PyEval_FrameClearAndPop`). + +This creates a "high-water mark" that always points to a frame still on the stack. +On subsequent samples, the profiler can walk from `current_frame` until it reaches +`last_profiled_frame`, knowing that frames from that point downward are unchanged +and can be retrieved from a cache. This significantly reduces the amount of remote +memory reads needed when call stacks are deep and stable at their base. + +The update in `_PyEval_FrameClearAndPop` is guarded: it only writes when +`last_profiled_frame` is non-NULL AND matches the frame being popped. This +prevents transient frames (called and returned between profiler samples) from +corrupting the cache pointer, while avoiding any overhead when profiling is inactive. + + ### The Instruction Pointer `_PyInterpreterFrame` has two fields which are used to maintain the instruction diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index 9c35684c945b3ea..0ef45ff8e02bc5f 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -107,7 +107,7 @@ As is explained later in the [Optimization: reusing fields to save memory](#optimization-reusing-fields-to-save-memory) section, these two extra fields are normally used to keep doubly linked lists of all the objects tracked by the garbage collector (these lists are the GC generations, more on -that in the [Optimization: incremental collection](#Optimization-incremental-collection) section), but +that in the [Optimization: generations](#Optimization-generations) section), but they are also reused to fulfill other purposes when the full doubly linked list structure is not needed as a memory optimization. @@ -153,7 +153,11 @@ pointer-sized (that is, eight bytes on a 64-bit platform). The garbage collector also temporarily repurposes the `ob_tid` (thread ID) and `ob_ref_local` (local reference count) fields for other purposes during -collections. +collections. The `ob_tid` field is later restored from the containing +mimalloc segment data structure. In some cases, such as when the original +allocating thread exits, this can result in a different `ob_tid` value. +Code should not rely on `ob_tid` being stable across operations that may +trigger garbage collection. C APIs @@ -199,22 +203,22 @@ unreachable: ```pycon >>> import gc ->>> +>>> >>> class Link: ... def __init__(self, next_link=None): ... self.next_link = next_link -... +... >>> link_3 = Link() >>> link_2 = Link(link_3) >>> link_1 = Link(link_2) >>> link_3.next_link = link_1 >>> A = link_1 >>> del link_1, link_2, link_3 ->>> +>>> >>> link_4 = Link() >>> link_4.next_link = link_4 >>> del link_4 ->>> +>>> >>> # Collect the unreachable Link object (and its .__dict__ dict). >>> gc.collect() 2 @@ -329,15 +333,16 @@ Once the GC knows the list of unreachable objects, a very delicate process start with the objective of completely destroying these objects. Roughly, the process follows these steps in order: -1. Handle and clear weak references (if any). Weak references to unreachable objects - are set to `None`. If the weak reference has an associated callback, the callback - is enqueued to be called once the clearing of weak references is finished. We only - invoke callbacks for weak references that are themselves reachable. If both the weak - reference and the pointed-to object are unreachable we do not execute the callback. - This is partly for historical reasons: the callback could resurrect an unreachable - object and support for weak references predates support for object resurrection. - Ignoring the weak reference's callback is fine because both the object and the weakref - are going away, so it's legitimate to say the weak reference is going away first. +1. Handle weak references with callbacks (if any). If the weak reference has + an associated callback, the callback is enqueued to be called after the weak + reference is cleared. We only invoke callbacks for weak references that + are themselves reachable. If both the weak reference and the pointed-to + object are unreachable we do not execute the callback. This is partly for + historical reasons: the callback could resurrect an unreachable object + and support for weak references predates support for object resurrection. + Ignoring the weak reference's callback is fine because both the object and + the weakref are going away, so it's legitimate to say the weak reference is + going away first. 2. If an object has legacy finalizers (`tp_del` slot) move it to the `gc.garbage` list. 3. Call the finalizers (`tp_finalize` slot) and mark the objects as already @@ -346,15 +351,21 @@ follows these steps in order: 4. Deal with resurrected objects. If some objects have been resurrected, the GC finds the new subset of objects that are still unreachable by running the cycle detection algorithm again and continues with them. -5. Call the `tp_clear` slot of every object so all internal links are broken and +5. Clear any weak references that still refer to unreachable objects. The + `wr_object` attribute for these weakrefs are set to `None`. Note that some + of these weak references maybe have been newly created during the running of + finalizers in step 3. Also, clear any weak references that are part of the + unreachable set. +6. Call the `tp_clear` slot of every object so all internal links are broken and the reference counts fall to 0, triggering the destruction of all unreachable objects. -Optimization: incremental collection -==================================== +Optimization: generations +========================= -In order to bound the length of each garbage collection pause, the GC implementation -for the default build uses incremental collection with two generations. +In order to limit the time each garbage collection takes, the GC +implementation for the default build uses a popular optimization: +generations. Generational garbage collection takes advantage of what is known as the weak generational hypothesis: Most objects die young. @@ -362,76 +373,29 @@ This has proven to be very close to the reality of many Python programs as many temporary objects are created and destroyed very quickly. To take advantage of this fact, all container objects are segregated into -two generations: young and old. Every new object starts in the young generation. -Each garbage collection scans the entire young generation and part of the old generation. - -The time taken to scan the young generation can be controlled by controlling its -size, but the size of the old generation cannot be controlled. -In order to keep pause times down, scanning of the old generation of the heap -occurs in increments. - -To keep track of what has been scanned, the old generation contains two lists: - -* Those objects that have not yet been scanned, referred to as the `pending` list. -* Those objects that have been scanned, referred to as the `visited` list. - -To detect and collect all unreachable objects in the heap, the garbage collector -must scan the whole heap. This whole heap scan is called a full scavenge. - -Increments ----------- - -Each full scavenge is performed in a series of increments. -For each full scavenge, the combined increments will cover the whole heap. - -Each increment is made up of: - -* The young generation -* The old generation's least recently scanned objects -* All objects reachable from those objects that have not yet been scanned this full scavenge - -The surviving objects (those that are not collected) are moved to the back of the -`visited` list in the old generation. - -When a full scavenge starts, no objects in the heap are considered to have been scanned, -so all objects in the old generation must be in the `pending` space. -When all objects in the heap have been scanned a cycle ends, and all objects are moved -to the `pending` list again. To avoid having to traverse the entire list, which list is -`pending` and which is `visited` is determined by a field in the `GCState` struct. -The `visited` and `pending` lists can be swapped by toggling this bit. - -Correctness ------------ - -The [algorithm for identifying cycles](#Identifying-reference-cycles) will find all -unreachable cycles in a list of objects, but will not find any cycles that are -even partly outside of that list. -Therefore, to be guaranteed that a full scavenge will find all unreachable cycles, -each cycle must be fully contained within a single increment. - -To make sure that no partial cycles are included in the increment we perform a -[transitive closure](https://en.wikipedia.org/wiki/Transitive_closure) -over reachable, unscanned objects from the initial increment. -Since the transitive closure of objects reachable from an object must be a (non-strict) -superset of any unreachable cycle including that object, we are guaranteed that a -transitive closure cannot contain any partial cycles. -We can exclude scanned objects, as they must have been reachable when scanned. -If a scanned object becomes part of an unreachable cycle after being scanned, it will -not be collected at this time, but it will be collected in the next full scavenge. +three spaces/generations. Every new +object starts in the first generation (generation 0). The previous algorithm is +executed only over the objects of a particular generation and if an object +survives a collection of its generation it will be moved to the next one +(generation 1), where it will be surveyed for collection less often. If +the same object survives another GC round in this new generation (generation 1) +it will be moved to the last generation (generation 2) where it will be +surveyed the least often. > [!NOTE] -> The GC implementation for the free-threaded build does not use incremental collection. -> Every collection operates on the entire heap. +> The GC implementation for the free-threaded build does not use generational +> collection. Every collection operates on the entire heap. + In order to decide when to run, the collector keeps track of the number of object allocations and deallocations since the last collection. When the number of allocations minus the number of deallocations exceeds `threshold0`, -collection starts. `threshold1` determines the fraction of the old -collection that is included in the increment. -The fraction is inversely proportional to `threshold1`, -as historically a larger `threshold1` meant that old generation -collections were performed less frequently. -`threshold2` is ignored. +collection starts. Initially only generation 0 is examined. If generation 0 has +been examined more than `threshold_1` times since generation 1 has been +examined, then generation 1 is examined as well. With generation 2, +things are a bit more complicated; see +[Collecting the oldest generation](#Collecting-the-oldest-generation) for +more information. These thresholds can be examined using the [`gc.get_threshold()`](https://docs.python.org/3/library/gc.html#gc.get_threshold) @@ -440,7 +404,7 @@ function: ```pycon >>> import gc >>> gc.get_threshold() -(700, 10, 10) +(2000, 10, 10) ``` The content of these generations can be examined using the @@ -453,84 +417,61 @@ specifically in a generation by calling `gc.collect(generation=NUM)`. ... pass ... >>> # Move everything to the old generation so it's easier to inspect ->>> # the young generation. +>>> # the younger generation. >>> gc.collect() 0 >>> # Create a reference cycle. >>> x = MyObj() >>> x.self = x ->>> ->>> # Initially the object is in the young generation. +>>> +>>> # Initially the object is in the youngest generation. >>> gc.get_objects(generation=0) [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] ->>> +>>> >>> # After a collection of the youngest generation the object ->>> # moves to the old generation. +>>> # moves to the next generation. >>> gc.collect(generation=0) 0 >>> gc.get_objects(generation=0) [] >>> gc.get_objects(generation=1) -[] ->>> gc.get_objects(generation=2) [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] ``` +Collecting the oldest generation +-------------------------------- + +In addition to the various configurable thresholds, the GC only triggers a full +collection of the oldest generation if the ratio `long_lived_pending / long_lived_total` +is above a given value (hardwired to 25%). The reason is that, while "non-full" +collections (that is, collections of the young and middle generations) will always +examine roughly the same number of objects (determined by the aforementioned +thresholds) the cost of a full collection is proportional to the total +number of long-lived objects, which is virtually unbounded. Indeed, it has +been remarked that doing a full collection every of object +creations entails a dramatic performance degradation in workloads which consist +of creating and storing lots of long-lived objects (for example, building a large list +of GC-tracked objects would show quadratic performance, instead of linear as +expected). Using the above ratio, instead, yields amortized linear performance +in the total number of objects (the effect of which can be summarized thusly: +"each full garbage collection is more and more costly as the number of objects +grows, but we do fewer and fewer of them"). + Optimization: excluding reachable objects ========================================= An object cannot be garbage if it can be reached. To avoid having to identify -reference cycles across the whole heap, we can reduce the amount of work done -considerably by first identifying objects reachable from objects known to be -alive. These objects are excluded from the normal cyclic detection process. - -The default and free-threaded build both implement this optimization but in -slightly different ways. - -Finding reachable objects for the default build GC --------------------------------------------------- - -This works by first moving most reachable objects to the `visited` space. -Empirically, most reachable objects can be reached from a small set of global -objects and local variables. This step does much less work per object, so -reduces the time spent performing garbage collection by at least half. - -> [!NOTE] -> Objects that are not determined to be reachable by this pass are not necessarily -> unreachable. We still need to perform the main algorithm to determine which objects -> are actually unreachable. -We use the same technique of forming a transitive closure as the incremental -collector does to find reachable objects, seeding the list with some global -objects and the currently executing frames. - -This phase moves objects to the `visited` space, as follows: - -1. All objects directly referred to by any builtin class, the `sys` module, the `builtins` -module and all objects directly referred to from stack frames are added to a working -set of reachable objects. -2. Until this working set is empty: - 1. Pop an object from the set and move it to the `visited` space - 2. For each object directly reachable from that object: - * If it is not already in `visited` space and it is a GC object, - add it to the working set - - -Before each increment of collection is performed, the stacks are scanned -to check for any new stack frames that have been created since the last -increment. All objects directly referred to from those stack frames are -added to the working set. -Then the above algorithm is repeated, starting from step 2. - +reference cycles across the whole heap, the free-threaded build first identifies +objects reachable from objects known to be alive. These objects are excluded +from the normal cyclic detection process. Finding reachable objects for the free-threaded GC -------------------------------------------------- Within the `gc_free_threading.c` implementation, this is known as the "mark -alive" pass or phase. It is similar in concept to what is done for the default -build GC. Rather than moving objects between double-linked lists, the -free-threaded GC uses a flag in `ob_gc_bits` to track if an object is -found to be definitely alive (not garbage). +alive" pass or phase. The free-threaded GC uses a flag in `ob_gc_bits` to track +if an object is found to be definitely alive (not garbage). To find objects reachable from known alive objects, known as the "roots", the `gc_mark_alive_from_roots()` function is used. Root objects include @@ -761,6 +702,14 @@ tuples can be untracked. A tuple can be untracked if all of its contents are already not tracked. Tuples are examined for untracking in all garbage collection cycles. +Dictionaries are always tracked from creation and are not untracked by the +garbage collector. Earlier versions (up to 3.13) used lazy tracking: empty or +atomic-only dicts were untracked on creation and re-tracked when a trackable +value was inserted (via `MAINTAIN_TRACKING`), and full collections called +`_PyDict_MaybeUntrack` to prune dicts whose values had become atomic. That +machinery was removed in 3.14 (GH-127010) because the per-set-item cost of +checking the tracking invariant outweighed the savings on full collections. + The garbage collector module provides the Python function `is_tracked(obj)`, which returns the current tracking status of the object. Subsequent garbage collections may change the tracking status of the object. diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md index 979a5b51521e4ef..fa652cd231cfa95 100644 --- a/InternalDocs/generators.md +++ b/InternalDocs/generators.md @@ -64,7 +64,7 @@ Iteration The [`FOR_ITER`](https://docs.python.org/dev/library/dis.html#opcode-FOR_ITER) instruction calls `__next__` on the iterator which is on the top of the stack, -and pushes the result to the stack. It has [`specializations`](adaptive.md) +and pushes the result to the stack. It has [`specializations`](interpreter.md) for a few common iterator types, including `FOR_ITER_GEN`, for iterating over a generator. `FOR_ITER_GEN` bypasses the call to `__next__`, and instead directly pushes the generator stack and resumes its execution from the diff --git a/InternalDocs/interpreter.md b/InternalDocs/interpreter.md index 38e9f6fced60880..7fc41a807dd5667 100644 --- a/InternalDocs/interpreter.md +++ b/InternalDocs/interpreter.md @@ -226,10 +226,11 @@ Up through 3.10, the call stack was implemented as a singly-linked list of heap allocation for the stack frame. Since 3.11, frames are no longer fully-fledged objects. Instead, a leaner internal -`_PyInterpreterFrame` structure is used, which is allocated using a custom allocator -function (`_PyThreadState_BumpFramePointer()`), which allocates and initializes a -frame structure. Usually a frame allocation is just a pointer bump, which improves -memory locality. +`_PyInterpreterFrame` structure is used. Most frames are allocated contiguously in a +per-thread stack (see `_PyThreadState_PushFrame` in [Python/pystate.c](../Python/pystate.c)), +which improves memory locality and reduces overhead. +If the current `datastack_chunk` has enough space (`_PyThreadState_HasStackSpace`) +then the lightweight `_PyFrame_PushUnchecked` can be used instead of `_PyThreadState_PushFrame`. Sometimes an actual `PyFrameObject` is needed, such as when Python code calls `sys._getframe()` or an extension module calls @@ -506,6 +507,38 @@ After the last `DEOPT_IF` has passed, a hit should be recorded with After an optimization has been deferred in the adaptive instruction, that should be recorded with `STAT_INC(BASE_INSTRUCTION, deferred)`. +## Interpreter types +There are three different types of interpreters to choose from based on compiler support: + + * traditional switch-case interpreter + + Supported by all compilers covered in PEP 7. + + * computed-gotos interpreter + + Enabled using configure option `--with-computed-gotos` and used by default on supported compilers. + It uses [Labels as Values](https://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html) + for more efficient dispatching. + + * tail-calling interpreter + + Enabled using configure option `--with-tail-call-interp` (or `--tail-call-interp` for build.bat on Windows). + It uses [tail calls](https://clang.llvm.org/docs/AttributeReference.html#musttail) and the + [preserve_none](https://clang.llvm.org/docs/AttributeReference.html#preserve-none) + calling convention between the small C functions that implement individual Python opcodes. + + Not all compilers support these and if they do not all targets might be supported (for example, + MSVC currently only supports x64 and only in optimized builds). + + In addition, compilers must do [escape analysis](https://gcc.gnu.org/onlinedocs/gcc/Common-Attributes.html#index-musttail) + of the lifetimes of automatic variables, function parameters, and temporaries to ensure proper tail-calls. They + emit a compile error in case of a violation or detection failure. The ability to detect this varies depending on the compiler and + also on the optimization level. Following techniques are particularly helpful to the MSVC compiler in this regard + * [Introducing additional scopes](https://github.com/python/cpython/blob/3908593039bde9d4b591ab09919003ee57418d64/Python/bytecodes.c#L2526) + * [extracting problematic code paths into a separate function](https://github.com/python/cpython/pull/143068/files#diff-729a985b0cb8b431cb291f1edb561bbbfea22e3f8c262451cd83328a0936a342R3724) + * [returning a pointer instead of taking it as an output parameter](https://github.com/python/cpython/blob/3908593039bde9d4b591ab09919003ee57418d64/Include/internal/pycore_ceval.h#L489-L492) + + Using `restrict` is another (currently unused) remedy. Additional resources -------------------- diff --git a/InternalDocs/jit.md b/InternalDocs/jit.md index c98ccb3ace8d9ec..1345f2db8b34db9 100644 --- a/InternalDocs/jit.md +++ b/InternalDocs/jit.md @@ -11,24 +11,54 @@ and this enables optimizations that span multiple instructions. Historically, the adaptive interpreter was referred to as `tier 1` and the JIT as `tier 2`. You will see remnants of this in the code. -## The Optimizer and Executors +## The Trace Recorder and Executors -The program begins running on the adaptive interpreter, until a `JUMP_BACKWARD` -instruction determines that it is "hot" because the counter in its +There are two interpreters in this section: + 1. Adaptive interpreter (the default behavior) + 2. Trace recording interpreter (enabled on JIT builds) + +The program begins running on the adaptive interpreter, until a `JUMP_BACKWARD` or +`RESUME` instruction determines that it is "hot" because the counter in its [inline cache](interpreter.md#inline-cache-entries) indicates that it executed more than some threshold number of times (see [`backoff_counter_triggers`](../Include/internal/pycore_backoff.h)). -It then calls the function `_PyOptimizer_Optimize()` in +It then calls the function `_PyJit_TryInitializeTracing` in [`Python/optimizer.c`](../Python/optimizer.c), passing it the current -[frame](frames.md) and instruction pointer. `_PyOptimizer_Optimize()` -constructs an object of type -[`_PyExecutorObject`](../Include/internal/pycore_optimizer.h) which implements -an optimized version of the instruction trace beginning at this jump. - -The optimizer determines where the trace ends, and the executor is set up +[frame](frames.md), instruction pointer and state. +The interpreter then switches into "tracing mode" via the macro +`ENTER_TRACING()`. On platforms that support computed goto and tail-calling +interpreters, the dispatch table is swapped out, while other platforms that do +not support either use a single flag in the opcode. +Execution between the normal interpreter and tracing interpreter are +interleaved via this dispatch mechanism. This means that while logically +there are two interpreters, the implementation appears to be a single +interpreter. + +During tracing mode, after each interpreter instruction's `DISPATCH()`, +the interpreter jumps to the `TRACE_RECORD` instruction. This instruction +records the previous instruction executed and also any live values of the next +operation it may require. It then translates the previous instruction to +a sequence of micro-ops using `_PyJit_translate_single_bytecode_to_trace`. +To ensure that the adaptive interpreter instructions +and cache entries are up-to-date, the trace recording interpreter always resets +the adaptive counters of adaptive instructions it sees. +This forces a re-specialization of any new instruction should an instruction +deoptimize. Thus, feeding the trace recorder up-to-date information. +Finally, the `TRACE_RECORD` instruction decides when to stop tracing +using various heuristics. + +Once trace recording concludes, `LEAVE_TRACING()` swaps out the dispatch +table/the opcode flag set earlier by `ENTER_TRACING()` is unset. +`stop_tracing_and_jit()` then calls `_PyOptimizer_Optimize()` which optimizes +the trace and constructs an +[`_PyExecutorObject`](../Include/internal/pycore_optimizer.h). + +JIT execution is set up to either return to the adaptive interpreter and resume execution, or transfer control to another executor (see `_PyExitData` in -Include/internal/pycore_optimizer.h). +Include/internal/pycore_optimizer.h). When resuming to the adaptive interpreter, +a "side exit", generated by an `EXIT_IF` may trigger recording of another trace. +While a "deopt", generated by a `DEOPT_IF`, does not trigger recording. The executor is stored on the [`code object`](code_objects.md) of the frame, in the `co_executors` field which is an array of executors. The start @@ -40,12 +70,7 @@ executor in `co_executors`. The micro-op (abbreviated `uop` to approximate `μop`) optimizer is defined in [`Python/optimizer.c`](../Python/optimizer.c) as `_PyOptimizer_Optimize`. -It translates an instruction trace into a sequence of micro-ops by replacing -each bytecode by an equivalent sequence of micro-ops (see -`_PyOpcode_macro_expansion` in -[pycore_opcode_metadata.h](../Include/internal/pycore_opcode_metadata.h) -which is generated from [`Python/bytecodes.c`](../Python/bytecodes.c)). -The micro-op sequence is then optimized by +It takes a micro-op sequence from the trace recorder and optimizes with `_Py_uop_analyze_and_optimize` in [`Python/optimizer_analysis.c`](../Python/optimizer_analysis.c) and an instance of `_PyUOpExecutor_Type` is created to contain it. @@ -53,7 +78,7 @@ and an instance of `_PyUOpExecutor_Type` is created to contain it. ## The JIT interpreter After a `JUMP_BACKWARD` instruction invokes the uop optimizer to create a uop -executor, it transfers control to this executor via the `GOTO_TIER_TWO` macro. +executor, it transfers control to this executor via the `TIER1_TO_TIER2` macro. CPython implements two executors. Here we describe the JIT interpreter, which is the simpler of them and is therefore useful for debugging and analyzing @@ -78,8 +103,8 @@ and execution returns to the adaptive interpreter. ## Invalidating Executors In addition to being stored on the code object, each executor is also -inserted into a list of all executors, which is stored in the interpreter -state's `executor_list_head` field. This list is used when it is necessary +inserted into contiguous arrays (`executor_blooms` and `executor_ptrs`) +stored in the interpreter state. These arrays are used when it is necessary to invalidate executors because values they used in their construction may have changed. @@ -89,7 +114,7 @@ When the full jit is enabled (python was configured with [`--enable-experimental-jit`](https://docs.python.org/dev/using/configure.html#cmdoption-enable-experimental-jit), the uop executor's `jit_code` field is populated with a pointer to a compiled C function that implements the executor logic. This function's signature is -defined by `jit_func` in [`pycore_jit.h`](Include/internal/pycore_jit.h). +defined by `jit_func` in [`pycore_jit.h`](../Include/internal/pycore_jit.h). When the executor is invoked by `ENTER_EXECUTOR`, instead of jumping to the uop interpreter at `tier2_dispatch`, the executor runs the function that `jit_code` points to. This function returns the instruction pointer diff --git a/InternalDocs/parser.md b/InternalDocs/parser.md index 1d0ffe6d40d078a..1bb4cdea5439f17 100644 --- a/InternalDocs/parser.md +++ b/InternalDocs/parser.md @@ -819,6 +819,13 @@ directory on the CPython repository and manually call the parser generator by ex $ python -m pegen python ``` +> [!CAUTION] +> Python's grammar (the `Grammar/python.gram` file) is written for the +> C backend. To experiment, you will need to write a grammar +> without C-specific parts like actions and the trailer. +> See [#133560](https://github.com/python/cpython/issues/133560) +> and [#96424](https://github.com/python/cpython/issues/96424) for more information. + This will generate a file called `parse.py` in the same directory that you can use to parse some input: diff --git a/InternalDocs/profiling_binary_format.md b/InternalDocs/profiling_binary_format.md new file mode 100644 index 000000000000000..7e4592a0d897055 --- /dev/null +++ b/InternalDocs/profiling_binary_format.md @@ -0,0 +1,541 @@ +# Profiling Binary Format + +The profiling module includes a binary file format for storing sampling +profiler data. This document describes the format's structure and the +design decisions behind it. + +The implementation is in +[`Modules/_remote_debugging/binary_io_writer.c`](../Modules/_remote_debugging/binary_io_writer.c) +and [`Modules/_remote_debugging/binary_io_reader.c`](../Modules/_remote_debugging/binary_io_reader.c), +with declarations in +[`Modules/_remote_debugging/binary_io.h`](../Modules/_remote_debugging/binary_io.h). + +## Overview + +The sampling profiler can generate enormous amounts of data. A typical +profiling session sampling at 1000 Hz for 60 seconds produces 60,000 samples. +Each sample contains a full call stack, often 20-50 frames deep, and each +frame includes a filename, function name, and line number. In a text-based +format like collapsed stacks, this would mean repeating the same long file +paths and function names thousands of times. + +The binary format addresses this through two key strategies: + +1. **Deduplication**: Strings and frames are stored once in lookup tables, + then referenced by small integer indices. A 100-character file path that + appears in 50,000 samples is stored once, not 50,000 times. + +2. **Compact encoding**: Variable-length integers (varints) encode small + values in fewer bytes. Since most indices are small (under 128), they + typically need only one byte instead of four. + +Together with optional zstd compression, these techniques reduce file sizes +by 10-50x compared to text formats while also enabling faster I/O. + +## File Layout + +The file consists of five sections: + +``` ++------------------+ Offset 0 +| Header | 64 bytes (fixed) ++------------------+ Offset 64 +| | +| Sample Data | Variable size (optionally compressed) +| | ++------------------+ string_table_offset +| String Table | Variable size ++------------------+ frame_table_offset +| Frame Table | Variable size ++------------------+ file_size - 32 +| Footer | 32 bytes (fixed) ++------------------+ file_size +``` + +The layout is designed for streaming writes during profiling. The profiler +cannot know in advance how many unique strings or frames will be encountered, +so these tables must be built incrementally and written at the end. + +The header comes first so readers can quickly validate the file and locate +the metadata tables. The sample data follows immediately, allowing the writer +to stream samples directly to disk (or through a compression stream) without +buffering the entire dataset in memory. + +The string and frame tables are placed after sample data because they grow +as new unique entries are discovered during profiling. By deferring their +output until finalization, the writer avoids the complexity of reserving +space or rewriting portions of the file. + +The footer at the end contains counts needed to allocate arrays before +parsing the tables. Placing it at a fixed offset from the end (rather than +at a variable offset recorded in the header) means readers can locate it +with a single seek to `file_size - 32`, without first reading the header. + +## Header + +``` + Offset Size Type Description ++--------+------+---------+----------------------------------------+ +| 0 | 4 | uint32 | Magic number (0x54414348 = "TACH") | +| 4 | 4 | uint32 | Format version | +| 8 | 4 | bytes | Python version (major, minor, micro, | +| | | | reserved) | +| 12 | 8 | uint64 | Start timestamp (microseconds) | +| 20 | 8 | uint64 | Sample interval (microseconds) | +| 28 | 4 | uint32 | Total sample count | +| 32 | 4 | uint32 | Thread count | +| 36 | 8 | uint64 | String table offset | +| 44 | 8 | uint64 | Frame table offset | +| 52 | 4 | uint32 | Compression type (0=none, 1=zstd) | +| 56 | 8 | bytes | Reserved (zero-filled) | ++--------+------+---------+----------------------------------------+ +``` + +The magic number `0x54414348` ("TACH" for Tachyon) identifies the file format +and also serves as an **endianness marker**. When read on a system with +different byte order than the writer, it appears as `0x48434154`. The reader +uses this to detect cross-endian files and automatically byte-swap all +multi-byte integer fields. + +The Python version field records the major, minor, and micro version numbers +of the Python interpreter that generated the file. This allows analysis tools +to detect version mismatches when replaying data collected on a different +Python version, which may have different internal structures or behaviors. + +The header is written as zeros initially, then overwritten with actual values +during finalization. This requires the output stream to be seekable, which +is acceptable since the format targets regular files rather than pipes or +network streams. + +## Sample Data + +Sample data begins at offset 64 and extends to `string_table_offset`. Samples +use delta compression to minimize redundancy when consecutive samples from the +same thread have identical or similar call stacks. + +### Stack Encoding Types + +Each sample record begins with thread identification, then an encoding byte: + +| Code | Name | Description | +|------|------|-------------| +| 0x00 | REPEAT | RLE: identical stack repeated N times | +| 0x01 | FULL | Complete stack (first sample or no match) | +| 0x02 | SUFFIX | Shares N frames from bottom of previous stack | +| 0x03 | POP_PUSH | Remove M frames from top, add N new frames | + +### Record Formats + +**REPEAT (0x00) - Run-Length Encoded Identical Stacks:** +``` ++-----------------+-----------+----------------------------------------+ +| thread_id | 8 bytes | Thread identifier (uint64, fixed) | +| interpreter_id | 4 bytes | Interpreter ID (uint32, fixed) | +| encoding | 1 byte | 0x00 (REPEAT) | +| count | varint | Number of samples in this RLE group | +| samples | varies | Interleaved: [delta: varint, status: 1]| +| | | repeated count times | ++-----------------+-----------+----------------------------------------+ +``` +The stack is inherited from this thread's previous sample. Each sample in the +group gets its own timestamp delta and status byte, stored as interleaved pairs +(delta1, status1, delta2, status2, ...) rather than separate arrays. + +**FULL (0x01) - Complete Stack:** +``` ++-----------------+-----------+----------------------------------------+ +| thread_id | 8 bytes | Thread identifier (uint64, fixed) | +| interpreter_id | 4 bytes | Interpreter ID (uint32, fixed) | +| encoding | 1 byte | 0x01 (FULL) | +| timestamp_delta | varint | Microseconds since thread's last sample| +| status | 1 byte | Thread state flags | +| stack_depth | varint | Number of frames in call stack | +| frame_indices | varint[] | Array of frame table indices | ++-----------------+-----------+----------------------------------------+ +``` +Used for the first sample from a thread, or when delta encoding would not +provide savings. + +**SUFFIX (0x02) - Shared Suffix Match:** +``` ++-----------------+-----------+----------------------------------------+ +| thread_id | 8 bytes | Thread identifier (uint64, fixed) | +| interpreter_id | 4 bytes | Interpreter ID (uint32, fixed) | +| encoding | 1 byte | 0x02 (SUFFIX) | +| timestamp_delta | varint | Microseconds since thread's last sample| +| status | 1 byte | Thread state flags | +| shared_count | varint | Frames shared from bottom of prev stack| +| new_count | varint | New frames at top of stack | +| new_frames | varint[] | Array of new_count frame indices | ++-----------------+-----------+----------------------------------------+ +``` +Used when a function call added frames to the top of the stack. The shared +frames from the previous stack are kept, and new frames are prepended. + +**POP_PUSH (0x03) - Pop and Push:** +``` ++-----------------+-----------+----------------------------------------+ +| thread_id | 8 bytes | Thread identifier (uint64, fixed) | +| interpreter_id | 4 bytes | Interpreter ID (uint32, fixed) | +| encoding | 1 byte | 0x03 (POP_PUSH) | +| timestamp_delta | varint | Microseconds since thread's last sample| +| status | 1 byte | Thread state flags | +| pop_count | varint | Frames to remove from top of prev stack| +| push_count | varint | New frames to add at top | +| new_frames | varint[] | Array of push_count frame indices | ++-----------------+-----------+----------------------------------------+ +``` +Used when the code path changed: some frames were popped (function returns) +and new frames were pushed (different function calls). + +### Thread and Interpreter Identification + +Thread IDs are 64-bit values that can be large (memory addresses on some +platforms) and vary unpredictably. Using a fixed 8-byte encoding avoids +the overhead of varint encoding for large values and simplifies parsing +since the reader knows exactly where each field begins. + +The interpreter ID identifies which Python sub-interpreter the thread +belongs to, allowing analysis tools to separate activity across interpreters +in processes using multiple sub-interpreters. + +### Status Byte + +The status byte is a bitfield encoding thread state at sample time: + +| Bit | Flag | Meaning | +|-----|-----------------------|--------------------------------------------| +| 0 | THREAD_STATUS_HAS_GIL | Thread holds the GIL (Global Interpreter Lock) | +| 1 | THREAD_STATUS_ON_CPU | Thread is actively running on a CPU core | +| 2 | THREAD_STATUS_UNKNOWN | Thread state could not be determined | +| 3 | THREAD_STATUS_GIL_REQUESTED | Thread is waiting to acquire the GIL | +| 4 | THREAD_STATUS_HAS_EXCEPTION | Thread has a pending exception | + +Multiple flags can be set simultaneously (e.g., a thread can hold the GIL +while also running on CPU). Analysis tools use these to filter samples or +visualize thread states over time. + +### Timestamp Delta Encoding + +Timestamps use delta encoding rather than absolute values. Absolute +timestamps in microseconds require 8 bytes each, but consecutive samples +from the same thread are typically separated by the sampling interval +(e.g., 1000 microseconds), so the delta between them is small and fits +in 1-2 varint bytes. The writer tracks the previous timestamp for each +thread separately. The first sample from a thread encodes its delta from +the profiling start time; subsequent samples encode the delta from that +thread's previous sample. This per-thread tracking is necessary because +samples are interleaved across threads in arrival order, not grouped by +thread. + +For REPEAT (RLE) records, timestamp deltas and status bytes are stored as +interleaved pairs (delta, status, delta, status, ...) - one pair per +repeated sample - allowing efficient batching while preserving the exact +timing and state of each sample. + +### Frame Indexing + +Each frame in a call stack is represented by an index into the frame table +rather than inline data. This provides massive space savings because call +stacks are highly repetitive: the same function appears in many samples +(hot functions), call stacks often share common prefixes (main -> app -> +handler -> ...), and recursive functions create repeated frame sequences. +A frame index is typically 1-2 varint bytes. Inline frame data would be +20-200+ bytes (two strings plus a line number). For a profile with 100,000 +samples averaging 30 frames each, this reduces frame data from potentially +gigabytes to tens of megabytes. + +Frame indices are written innermost-first (the currently executing frame +has index 0 in the array). This ordering works well with delta compression: +function calls typically add frames at the top (index 0), while shared +frames remain at the bottom. + +## String Table + +The string table stores deduplicated UTF-8 strings (filenames and function +names). It begins at `string_table_offset` and contains entries in order of +their assignment during writing: + +``` ++----------------+ +| length: varint | +| data: bytes | ++----------------+ (repeated for each string) +``` + +Strings are stored in the order they were first encountered during writing. +The first unique filename gets index 0, the second gets index 1, and so on. +Length-prefixing (rather than null-termination) allows strings containing +null bytes and enables readers to allocate exact-sized buffers. The varint +length encoding means short strings (under 128 bytes) need only one length +byte. + +## Frame Table + +The frame table stores deduplicated frame entries with full source position +information and bytecode opcode: + +``` ++----------------------------+ +| filename_idx: varint | +| funcname_idx: varint | +| lineno: svarint | +| end_lineno_delta: svarint | +| column: svarint | +| end_column_delta: svarint | +| opcode: u8 | ++----------------------------+ (repeated for each frame) +``` + +### Field Definitions + +| Field | Type | Description | +|------------------|---------------|----------------------------------------------------------| +| filename_idx | varint | Index into string table for file name | +| funcname_idx | varint | Index into string table for function name | +| lineno | zigzag varint | Start line number (-1 for synthetic frames) | +| end_lineno_delta | zigzag varint | Delta from lineno (end_lineno = lineno + delta) | +| column | zigzag varint | Start column offset in UTF-8 bytes (-1 if not available) | +| end_column_delta | zigzag varint | Delta from column (end_column = column + delta) | +| opcode | u8 | Python bytecode opcode (0-254) or 255 for None | + +### Delta Encoding + +Position end values use delta encoding for efficiency: + +- `end_lineno = lineno + end_lineno_delta` +- `end_column = column + end_column_delta` + +Typical values: +- `end_lineno_delta`: Usually 0 (single-line expressions) → encodes to 1 byte +- `end_column_delta`: Usually 5-20 (expression width) → encodes to 1 byte + +This saves ~1-2 bytes per frame compared to absolute encoding. When the base +value (lineno or column) is -1 (not available), the delta is stored as 0 and +the reconstructed value is -1. + +### Sentinel Values + +- `opcode = 255`: No opcode captured +- `lineno = -1`: Synthetic frame (no source location) +- `column = -1`: Column offset not available + +### Deduplication + +Each unique (filename, funcname, lineno, end_lineno, column, end_column, +opcode) combination gets one entry. This enables instruction-level profiling +where multiple bytecode instructions on the same line can be distinguished. + +Strings and frames are deduplicated separately because they have different +cardinalities and reference patterns. A codebase might have hundreds of +unique source files but thousands of unique functions. Many functions share +the same filename, so storing the filename index in each frame entry (rather +than the full string) provides an additional layer of deduplication. A frame +entry is typically 7-9 bytes rather than two full strings plus location data. + +### Size Analysis + +Typical frame size with delta encoding: +- file_idx: 1-2 bytes +- func_idx: 1-2 bytes +- lineno: 1-2 bytes +- end_lineno_delta: 1 byte (usually 0) +- column: 1 byte (usually < 64) +- end_column_delta: 1 byte (usually < 64) +- opcode: 1 byte + +**Total: ~7-9 bytes per frame** + +Line numbers and columns use signed varint (zigzag encoding) to handle +sentinel values efficiently. Synthetic frames—generated frames that don't +correspond directly to Python source code, such as C extension boundaries or +internal interpreter frames—use -1 to indicate the absence of a source +location. Zigzag encoding ensures these small negative values encode +efficiently (−1 becomes 1, which is one byte) rather than requiring the +maximum varint length. + +## Footer + +``` + Offset Size Type Description ++--------+------+---------+----------------------------------------+ +| 0 | 4 | uint32 | String count | +| 4 | 4 | uint32 | Frame count | +| 8 | 8 | uint64 | Total file size | +| 16 | 16 | bytes | Checksum (reserved, currently zeros) | ++--------+------+---------+----------------------------------------+ +``` + +The string and frame counts allow readers to pre-allocate arrays of the +correct size before parsing the tables. Without these counts, readers would +need to either scan the tables twice (once to count, once to parse) or use +dynamically-growing arrays. + +The file size field provides a consistency check: if the actual file size +does not match, the file may be truncated or corrupted. + +The checksum field is reserved for future use. A checksum would allow +detection of corruption but adds complexity and computation cost. The +current implementation leaves this as zeros. + +## Variable-Length Integer Encoding + +The format uses LEB128 (Little Endian Base 128) for unsigned integers and +zigzag + LEB128 for signed integers. These encodings are widely used +(Protocol Buffers, DWARF debug info, WebAssembly) and well-understood. + +### Unsigned Varint (LEB128) + +Each byte stores 7 bits of data. The high bit indicates whether more bytes +follow: + +``` +Value Encoded bytes +0-127 [0xxxxxxx] (1 byte) +128-16383 [1xxxxxxx] [0xxxxxxx] (2 bytes) +16384+ [1xxxxxxx] [1xxxxxxx] ... (3+ bytes) +``` + +Most indices in profiling data are small. A profile with 1000 unique frames +needs at most 2 bytes per frame index. The common case (indices under 128) +needs only 1 byte. + +### Signed Varint (Zigzag) + +Standard LEB128 encodes −1 as a very large unsigned value, requiring many +bytes. Zigzag encoding interleaves positive and negative values: + +``` + 0 -> 0 -1 -> 1 1 -> 2 -2 -> 3 2 -> 4 +``` + +This ensures small-magnitude values (whether positive or negative) encode +in few bytes. + +## Compression + +When compression is enabled, the sample data region contains a zstd stream. +The string table, frame table, and footer remain uncompressed so readers can +access metadata without decompressing the entire file. A tool that only needs +to report "this file contains 50,000 samples of 3 threads" can read the header +and footer without touching the compressed sample data. This also simplifies +the format: the header's offset fields point directly to the tables rather +than to positions within a decompressed stream. + +Zstd provides an excellent balance of compression ratio and speed. Profiling +data compresses very well (often 5-10x) due to repetitive patterns: the same +small set of frame indices appears repeatedly, and delta-encoded timestamps +cluster around the sampling interval. Zstd's streaming API allows compression +without buffering the entire dataset. The writer feeds sample data through +the compressor incrementally, flushing compressed chunks to disk as they +become available. + +Level 5 compression is used as a default. Lower levels (1-3) are faster but +compress less; higher levels (6+) compress more but slow down writing. Level +5 provides good compression with minimal impact on profiling overhead. + +## Reading and Writing + +### Writing + +1. Open the output file and write 64 zero bytes as a placeholder header +2. Initialize empty string and frame dictionaries for deduplication +3. For each sample: + - Intern any new strings, assigning sequential indices + - Intern any new frames, assigning sequential indices + - Encode the sample record and write to the buffer + - Flush the buffer through compression (if enabled) when full +4. Flush remaining buffered data and finalize compression +5. Write the string table (length-prefixed strings in index order) +6. Write the frame table (varint-encoded entries in index order) +7. Write the footer with final counts +8. Seek to offset 0 and write the header with actual values + +The writer maintains two dictionaries: one mapping strings to indices, one +mapping (filename_idx, funcname_idx, lineno) tuples to frame indices. These +enable O(1) lookup during interning. + +### Reading + +1. Read the header magic number to detect endianness (set `needs_swap` flag + if the magic appears byte-swapped) +2. Validate version and read remaining header fields (byte-swapping if needed) +3. Seek to end − 32 and read the footer (byte-swapping counts if needed) +4. Allocate string array of `string_count` elements +5. Parse the string table, populating the array +6. Allocate frame array of `frame_count * 3` uint32 elements +7. Parse the frame table, populating the array +8. If compressed, decompress the sample data region +9. Iterate through samples, resolving indices to strings/frames + (byte-swapping thread_id and interpreter_id if needed) + +The reader builds lookup arrays rather than dictionaries since it only needs +index-to-value mapping, not value-to-index. + +## Platform Considerations + +### Byte Ordering and Cross-Platform Portability + +The binary format uses **native byte order** for all multi-byte integer +fields when writing. However, the reader supports **cross-endian reading**: +files written on a little-endian system (x86, ARM) can be read on a +big-endian system (s390x, PowerPC), and vice versa. + +The magic number doubles as an endianness marker. When read on a system with +different byte order, it appears byte-swapped (`0x48434154` instead of +`0x54414348`). The reader detects this and automatically byte-swaps all +fixed-width integer fields during parsing. + +Writers must use `memcpy()` from properly-sized integer types when writing +fixed-width integer fields. When the source variable's type differs from the +field width (e.g., `size_t` written as 4 bytes), explicit casting to the +correct type (e.g., `uint32_t`) is required before `memcpy()`. On big-endian +systems, copying from an oversized type would copy the wrong bytes—high-order +zeros instead of the actual value. + +The reader tracks whether byte-swapping is needed via a `needs_swap` flag set +during header parsing. All fixed-width fields in the header, footer, and +sample data are conditionally byte-swapped using Python's internal byte-swap +functions (`_Py_bswap32`, `_Py_bswap64` from `pycore_bitutils.h`). + +Variable-length integers (varints) are byte-order independent since they +encode values one byte at a time using the LEB128 scheme, so they require +no special handling for cross-endian reading. + +### Memory-Mapped I/O + +On Unix systems (Linux, macOS), the reader uses `mmap()` to map the file +into the process address space. The kernel handles paging data in and out +as needed, no explicit read() calls or buffer management are required, +multiple readers can share the same physical pages, and sequential access +patterns benefit from kernel read-ahead. + +The implementation uses `madvise()` to hint the access pattern to the kernel: +`MADV_SEQUENTIAL` indicates the file will be read linearly, enabling +aggressive read-ahead. `MADV_WILLNEED` requests pre-faulting of pages. +On Linux, `MAP_POPULATE` pre-faults all pages at mmap time rather than on +first access, moving page fault overhead from the parsing loop to the +initial mapping for more predictable performance. For large files (over +32 MB), `MADV_HUGEPAGE` requests transparent huge pages (2 MB instead of +4 KB) to reduce TLB pressure when accessing large amounts of data. + +On Windows, the implementation falls back to standard file I/O with full +file buffering. Profiling data files are typically small enough (tens to +hundreds of megabytes) that this is acceptable. + +The writer uses a 512 KB buffer to batch small writes. Each sample record +is typically tens of bytes; writing these individually would incur excessive +syscall overhead. The buffer accumulates data until full, then flushes in +one write() call (or feeds through the compression stream). + +## Future Considerations + +The format reserves space for future extensions. The 12 reserved bytes in +the header could hold additional metadata. The 16-byte checksum field in +the footer is currently unused. The version field allows incompatible +changes with graceful rejection. New compression types could be added +(compression_type > 1). + +Any changes that alter the meaning of existing fields or the parsing logic +should increment the version number to prevent older readers from +misinterpreting new files. diff --git a/InternalDocs/qsbr.md b/InternalDocs/qsbr.md index 1c4a79a7b44436a..d511396fdab6452 100644 --- a/InternalDocs/qsbr.md +++ b/InternalDocs/qsbr.md @@ -101,15 +101,39 @@ Periodically, a polling mechanism processes this deferred-free list: To reduce memory contention from frequent updates to the global `wr_seq`, its advancement is sometimes deferred. Instead of incrementing `wr_seq` on every -reclamation request, each thread tracks its number of deferrals locally. Once -the deferral count reaches a limit (QSBR_DEFERRED_LIMIT, currently 10), the -thread advances the global `wr_seq` and resets its local count. - -When an object is added to the deferred-free list, its qsbr_goal is set to -`wr_seq` + 2. By setting the goal to the next sequence value, we ensure it's safe -to defer the global counter advancement. This optimization improves runtime -speed but may increase peak memory usage by slightly delaying when memory can -be reclaimed. +reclamation request, the object's qsbr_goal is set to `wr_seq` + 2 (the value +the counter *would* take on its next advance) without actually advancing the +global counter. This is safe because the goal still corresponds to a future +sequence value that no thread has yet observed as quiescent. + +Whether to actually advance `wr_seq` is decided per request, based on how +much memory and how many items the calling thread has already deferred since +its last advance: + +* For deferred object frees (`_PyMem_FreeDelayed`), the thread tracks both a + count (`deferred_count`) and an estimate of the held memory + (`deferred_memory`). The global `wr_seq` is advanced when the freed block + is larger than `QSBR_FREE_MEM_LIMIT` (1 MiB), when the accumulated deferred + memory exceeds that limit, or when the count exceeds `QSBR_DEFERRED_LIMIT` + (127, sized so a chunk of work items is processed before it overflows). + Crossing any of these thresholds also sets a per-thread `should_process` + flag, signalling that the deferred-free list should be drained. + +* For mimalloc pages held by QSBR, the thread tracks `deferred_page_memory` + and advances `wr_seq` when either the individual page or the accumulated + page memory exceeds `QSBR_PAGE_MEM_LIMIT` (4096 * 20 bytes). Advancing + promptly here matters because a held page cannot be reused for a different + size class or by a different thread. + +Processing of the deferred-free list normally happens from the eval breaker +(rather than from inside `_PyMem_FreeDelayed`), which gives the global +`rd_seq` a better chance to have advanced far enough that items can actually +be freed. `_PyMem_ProcessDelayed` is still called from the free path as a +safety valve when a work-item chunk fills up. + +This optimization improves runtime speed but may increase peak memory usage +by slightly delaying when memory can be reclaimed; the size-based thresholds +above bound that extra memory. ## Limitations diff --git a/InternalDocs/stack_protection.md b/InternalDocs/stack_protection.md new file mode 100644 index 000000000000000..14802e57d095f4d --- /dev/null +++ b/InternalDocs/stack_protection.md @@ -0,0 +1,68 @@ +# Stack Protection + +CPython protects against stack overflow in the form of runaway, or just very deep, recursion by raising a `RecursionError` instead of just crashing. +Protection against pure Python stack recursion has existed since very early, but in 3.12 we added protection against stack overflow +in C code. This was initially implemented using a counter and later improved in 3.14 to use the actual stack depth. +For those platforms that support it (Windows, Mac, and most Linuxes) we query the operating system to find the stack bounds. +For other platforms we use conservative estimates. + + +The C stack looks like this: + +``` + +-------+ <--- Top of machine stack + | | + | | + + ~~ + + | | + | | + +-------+ <--- Soft limit + | | + | | _PyOS_STACK_MARGIN_BYTES + | | + +-------+ <--- Hard limit + | | + | | _PyOS_STACK_MARGIN_BYTES + | | + +-------+ <--- Bottom of machine stack +``` + + +We get the current stack pointer using compiler intrinsics where available, or by taking the address of a C local variable. See `_Py_get_machine_stack_pointer()`. + +The soft and hard limits pointers are set by calling `_Py_InitializeRecursionLimits()` during thread initialization. + +Recursion checks are performed by `_Py_EnterRecursiveCall()` or `_Py_EnterRecursiveCallTstate()` which compare the stack pointer to the soft limit. If the stack pointer is lower than the soft limit, then `_Py_CheckRecursiveCall()` is called which checks against both the hard and soft limits: + +```python +kb_used = (stack_top - stack_pointer)>>10 +if stack_pointer < bottom_of_machine_stack: + pass # Our stack limits could be wrong so it is safest to do nothing. +elif stack_pointer < hard_limit: + FatalError(f"Unrecoverable stack overflow (used {kb_used} kB)") +elif stack_pointer < soft_limit: + raise RecursionError(f"Stack overflow (used {kb_used} kB)") +``` + +### User space threads and other oddities + +Some libraries provide user-space threads. These will change the C stack at runtime. +To guard against this we only raise if the stack pointer is in the window between the expected stack base and the soft limit. + +### Diagnosing and fixing stack overflows + +For stack protection to work correctly the amount of stack consumed between calls to `_Py_EnterRecursiveCall()` must be less than `_PyOS_STACK_MARGIN_BYTES`. + +If you see a traceback ending in: `RecursionError: Stack overflow (used ... kB)` then the stack protection is working as intended. If you don't expect to see the error, then check the amount of stack used. If it seems low then CPython may not be configured properly. + +However, if you see a fatal error or crash, then something is not right. +Either a recursive call is not checking `_Py_EnterRecursiveCall()`, or the amount of C stack consumed by a single call exceeds `_PyOS_STACK_MARGIN_BYTES`. If a hard crash occurs, it probably means that the amount of C stack consumed is more than double `_PyOS_STACK_MARGIN_BYTES`. + +Likely causes: +* Recursive code is not calling `_Py_EnterRecursiveCall()` +* `-O0` compilation flags, especially for Clang. With no optimization, C calls can consume a lot of stack space +* Giant, complex functions in third-party C extensions. This is unlikely as the function in question would need to be more complicated than the bytecode interpreter. +* `_PyOS_STACK_MARGIN_BYTES` is just too low. +* `_Py_InitializeRecursionLimits()` is not setting the soft and hard limits correctly for that platform. diff --git a/InternalDocs/stackrefs.md b/InternalDocs/stackrefs.md new file mode 100644 index 000000000000000..5774be9c56d3635 --- /dev/null +++ b/InternalDocs/stackrefs.md @@ -0,0 +1,79 @@ +# Stack references (`_PyStackRef`) + +Stack references are the interpreter's tagged representation of values on the evaluation stack. +They carry metadata to track ownership and support optimizations such as tagged small ints. + +## Shape and tagging + +- A `_PyStackRef` is a tagged pointer-sized value (see `Include/internal/pycore_stackref.h`). +- Tag bits distinguish three cases: + - `Py_TAG_REFCNT` unset - reference count lives on the pointed-to object. + - `Py_TAG_REFCNT` set - ownership is "borrowed" (no refcount to drop on close) or the object is immortal. + - `Py_INT_TAG` set - tagged small integer stored directly in the stackref (no heap allocation). +- Special constants: `PyStackRef_NULL`, `PyStackRef_ERROR`, and embedded `None`/`True`/`False`. + +In GIL builds, most objects carry their refcount; tagged borrowed refs skip decref on close. In free +threading builds, the tag is also used to mark deferred refcounted objects so the GC can see them and +to avoid refcount contention on commonly shared objects. + +## Converting to and from PyObject* + +Three conversions control ownership: + +- `PyStackRef_FromPyObjectNew(obj)` - create a new reference (INCREF if mortal). +- `PyStackRef_FromPyObjectSteal(obj)` - take over ownership without changing the count unless the + object is immortal. +- `PyStackRef_FromPyObjectBorrow(obj)` - create a borrowed stackref (never decref on close). + +The `obj` argument must not be `NULL`. + +Going back to `PyObject*` mirrors this: + +- `PyStackRef_AsPyObjectBorrow(ref)` - borrow the underlying pointer +- `PyStackRef_AsPyObjectSteal(ref)` - transfer ownership from the stackref; if ref is borrowed or + deferred, this creates a new owning `PyObject*` reference. +- `PyStackRef_AsPyObjectNew(ref)` - create a new owning reference + +Only `PyStackRef_AsPyObjectBorrow` allows ref to be `PyStackRef_NULL`. + +## Operations on stackrefs + +The interpreter treats `_PyStackRef` as the unit of stack storage. Ownership must be managed with +the stackref primitives: + +- `PyStackRef_DUP` - like `Py_NewRef` for stackrefs; preserves the original. +- `PyStackRef_Borrow` - create a borrowed stackref from another stackref. +- `PyStackRef_CLOSE` / `PyStackRef_XCLOSE` - like `Py_DECREF`; invalidates the stackref. +- `PyStackRef_CLEAR` - like `Py_CLEAR`; closes and sets the stackref to `PyStackRef_NULL` +- `PyStackRef_MakeHeapSafe` - converts borrowed reference to owning reference + +Borrow tracking (for debug builds with `Py_STACKREF_DEBUG`) records who you borrowed from and reports +double-close, leaked borrows, or use-after-close via fatal errors. + +## Borrow-friendly opcodes + +The interpreter can push borrowed references directly. For example, `LOAD_FAST_BORROW` loads a local +variable as a borrowed `_PyStackRef`, avoiding both INCREF and DECREF for the temporary lifetime on +the evaluation stack. + +## Tagged integers on the stack + +Small ints can be stored inline with `Py_INT_TAG`, so no heap object is involved. Helpers like +`PyStackRef_TagInt`, `PyStackRef_UntagInt`, and `PyStackRef_IncrementTaggedIntNoOverflow` operate on +these values. Type checks use `PyStackRef_IsTaggedInt` and `PyStackRef_LongCheck`. + +## Free threading considerations + +Objects that support deferred reference counting can be pushed to the evaluation +stack and stored in local variables without directly incrementing the reference +count because they are only freed during cyclic garbage collection. This avoids +reference count contention on commonly shared objects such as methods and types. The GC +scans each thread's locals and evaluation stack to keep objects that use +deferred reference counting alive. + +## Debugging support + +`Py_STACKREF_DEBUG` builds replace the inline tags with table-backed IDs so the runtime can track +creation sites, borrows, closes, and leaks. Enabling `Py_STACKREF_CLOSE_DEBUG` additionally records +double closes. The tables live on `PyInterpreterState` and are initialized in `pystate.c`; helper +routines reside in `Python/stackrefs.c`. diff --git a/InternalDocs/string_interning.md b/InternalDocs/string_interning.md index 26a5197c6e70f38..829a27654a37d34 100644 --- a/InternalDocs/string_interning.md +++ b/InternalDocs/string_interning.md @@ -16,8 +16,8 @@ dynamic interning. The 256 possible one-character latin-1 strings, which can be retrieved with `_Py_LATIN1_CHR(c)`, are stored in statically allocated arrays, -`_PyRuntime.static_objects.strings.ascii` and -`_PyRuntime.static_objects.strings.latin1`. +`_PyRuntime.static_objects.singletons.strings.ascii` and +`_PyRuntime.static_objects.singletons.strings.latin1`. Longer singleton strings are marked in C source with `_Py_ID` (if the string is a valid C identifier fragment) or `_Py_STR` (if it needs a separate @@ -52,15 +52,9 @@ The key and value of each entry in this dict reference the same object. ## Immortality and reference counting -Invariant: Every immortal string is interned. +In the GIL-enabled build interned strings may be mortal or immortal. In the +free-threaded build, interned strings are always immortal. -In practice, this means that you must not use `_Py_SetImmortal` on -a string. (If you know it's already immortal, don't immortalize it; -if you know it's not interned you might be immortalizing a redundant copy; -if it's interned and mortal it needs extra processing in -`_PyUnicode_InternImmortal`.) - -The converse is not true: interned strings can be mortal. For mortal interned strings: - the 2 references from the interned dict (key & value) are excluded from diff --git a/InternalDocs/structure.md b/InternalDocs/structure.md new file mode 100644 index 000000000000000..75c8476aa0ad989 --- /dev/null +++ b/InternalDocs/structure.md @@ -0,0 +1,40 @@ +# CPython source code + +This section gives an overview of CPython's code structure and provides +a summary of file locations for modules and built-ins. + + +## Source code layout + +For a Python module, the typical layout is: + +* `Lib/.py` +* `Modules/_.c` (if there's also a C accelerator module) +* `Lib/test/test_.py` +* `Doc/library/.rst` + +For an extension module, the typical layout is: + +* `Modules/module.c` +* `Lib/test/test_.py` +* `Doc/library/.rst` + +For builtin types, the typical layout is: + +* `Objects/object.c` +* `Lib/test/test_.py` +* [`Doc/library/stdtypes.rst`](../Doc/library/stdtypes.rst) + +For builtin functions, the typical layout is: + +* [`Python/bltinmodule.c`](../Python/bltinmodule.c) +* [`Lib/test/test_builtin.py`](../Lib/test/test_builtin.py) +* [`Doc/library/functions.rst`](../Doc/library/functions.rst) + +Some exceptions to these layouts are: + +* built-in type `int` is at [`Objects/longobject.c`](../Objects/longobject.c) +* built-in type `str` is at [`Objects/unicodeobject.c`](../Objects/unicodeobject.c) +* built-in module `sys` is at [`Python/sysmodule.c`](../Python/sysmodule.c) +* built-in module `marshal` is at [`Python/marshal.c`](../Python/marshal.c) +* Windows-only module `winreg` is at [`PC/winreg.c`](../PC/winreg.c) diff --git a/Lib/.ruff.toml b/Lib/.ruff.toml new file mode 100644 index 000000000000000..c9ae9b95641b318 --- /dev/null +++ b/Lib/.ruff.toml @@ -0,0 +1,21 @@ +extend = "../.ruff.toml" # Inherit the project-wide settings + +# Unlike Tools/, stdlib can use newer syntax than PYTHON_FOR_REGEN +target-version = "py315" + +[lint] +select = [ + "F401", # Unused import +] + +[lint.per-file-ignores] +"ctypes/__init__.py" = ["F401"] # Re-exports from _ctypes +"ensurepip/__init__.py" = ["F401"] # `import zlib` availability check +"idlelib/idle_test/htest.py" = ["F401"] # Import for Windows DPI side effect +"idlelib/idle_test/test_iomenu.py" = ["F401"] # Imports checked for existence +"importlib/_abc.py" = ["F401"] # Bootstrap-sensitive _bootstrap import +"importlib/machinery.py" = ["F401"] # NamespacePath re-export +"importlib/metadata/__init__.py" = ["F401"] # Synced from importlib_metadata +"profiling/sampling/sample.py" = ["F401"] # Re-exports PROFILING_MODE_* constants +"ssl.py" = ["F401"] # Re-exports from _ssl +"warnings.py" = ["F401"] # Re-exports from _py_warnings diff --git a/Lib/_android_support.py b/Lib/_android_support.py index ae506f6a4b57b89..320dab52acdc0bd 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -29,15 +29,19 @@ def init_streams(android_log_write, stdout_prio, stderr_prio): global logcat logcat = Logcat(android_log_write) - - sys.stdout = TextLogStream( - stdout_prio, "python.stdout", sys.stdout.fileno()) - sys.stderr = TextLogStream( - stderr_prio, "python.stderr", sys.stderr.fileno()) + sys.stdout = TextLogStream(stdout_prio, "python.stdout", sys.stdout) + sys.stderr = TextLogStream(stderr_prio, "python.stderr", sys.stderr) class TextLogStream(io.TextIOWrapper): - def __init__(self, prio, tag, fileno=None, **kwargs): + def __init__(self, prio, tag, original=None, **kwargs): + # Respect the -u option. + if original: + kwargs.setdefault("write_through", original.write_through) + fileno = original.fileno() + else: + fileno = None + # The default is surrogateescape for stdout and backslashreplace for # stderr, but in the context of an Android log, readability is more # important than reversibility. @@ -164,6 +168,13 @@ def write(self, prio, tag, message): # message. message = message.replace(b"\x00", b"\xc0\x80") + # On API level 30 and higher, Logcat will strip any number of leading + # newlines. This is visible in all `logcat` modes, even --binary. Work + # around this by adding a leading space, which shouldn't make any + # difference to the log's usability. + if message.startswith(b"\n"): + message = b" " + message + with self._lock: now = time() self._bucket_level += ( diff --git a/Lib/_ast_unparse.py b/Lib/_ast_unparse.py index 16cf56f62cc1e58..916bb25d74dee9b 100644 --- a/Lib/_ast_unparse.py +++ b/Lib/_ast_unparse.py @@ -239,11 +239,11 @@ def visit_NamedExpr(self, node): self.traverse(node.value) def visit_Import(self, node): - self.fill("import ") + self.fill("lazy import " if node.is_lazy else "import ") self.interleave(lambda: self.write(", "), self.traverse, node.names) def visit_ImportFrom(self, node): - self.fill("from ") + self.fill("lazy from " if node.is_lazy else "from ") self.write("." * (node.level or 0)) if node.module: self.write(node.module) @@ -658,9 +658,9 @@ def _unparse_interpolation_value(self, inner): unparser.set_precedence(_Precedence.TEST.next(), inner) return unparser.visit(inner) - def _write_interpolation(self, node, is_interpolation=False): + def _write_interpolation(self, node, use_str_attr=False): with self.delimit("{", "}"): - if is_interpolation: + if use_str_attr: expr = node.str else: expr = self._unparse_interpolation_value(node.value) @@ -678,7 +678,8 @@ def visit_FormattedValue(self, node): self._write_interpolation(node) def visit_Interpolation(self, node): - self._write_interpolation(node, is_interpolation=True) + # If `str` is set to `None`, use the `value` to generate the source code. + self._write_interpolation(node, use_str_attr=node.str is not None) def visit_Name(self, node): self.write(node.id) @@ -737,9 +738,13 @@ def visit_SetComp(self, node): def visit_DictComp(self, node): with self.delimit("{", "}"): - self.traverse(node.key) - self.write(": ") - self.traverse(node.value) + if node.value: + self.traverse(node.key) + self.write(": ") + self.traverse(node.value) + else: + self.write("**") + self.traverse(node.key) for gen in node.generators: self.traverse(gen) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 51263d696a17773..0e1d8ccf44a4351 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -461,8 +461,8 @@ def __subclasshook__(cls, C): class _CallableGenericAlias(GenericAlias): """ Represent `Callable[argtypes, resulttype]`. - This sets ``__args__`` to a tuple containing the flattened ``argtypes`` - followed by ``resulttype``. + This sets ``__args__`` to a tuple containing the flattened + ``argtypes`` followed by ``resulttype``. Example: ``Callable[[int, str], float]`` sets ``__args__`` to ``(int, str, float)``. @@ -823,6 +823,7 @@ def __eq__(self, other): __reversed__ = None +Mapping.register(frozendict) Mapping.register(mappingproxy) Mapping.register(framelocalsproxy) @@ -927,8 +928,9 @@ def __delitem__(self, key): __marker = object() def pop(self, key, default=__marker): - '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. + '''D.pop(k[,d]) -> v, remove specified key and return the corresponding + value. If key is not found, d is returned if given, otherwise + KeyError is raised. ''' try: value = self[key] @@ -962,9 +964,12 @@ def clear(self): def update(self, other=(), /, **kwds): ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. - If E present and has a .keys() method, does: for k in E.keys(): D[k] = E[k] - If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v - In either case, this is followed by: for k, v in F.items(): D[k] = v + If E present and has a .keys() method, does: + for k in E.keys(): D[k] = E[k] + If E present and lacks .keys() method, does: + for (k, v) in E: D[k] = v + In either case, this is followed by: + for k, v in F.items(): D[k] = v ''' if isinstance(other, Mapping): for key in other: @@ -1029,8 +1034,8 @@ def __reversed__(self): yield self[i] def index(self, value, start=0, stop=None): - '''S.index(value, [start, [stop]]) -> integer -- return first index of value. - Raises ValueError if the value is not present. + '''S.index(value, [start, [stop]]) -> integer -- return first index of + value. Raises ValueError if the value is not present. Supporting start and stop arguments is optional, but recommended. @@ -1061,6 +1066,41 @@ def count(self, value): Sequence.register(range) Sequence.register(memoryview) +class _DeprecateByteStringMeta(ABCMeta): + def __new__(cls, name, bases, namespace, **kwargs): + if name != "ByteString": + import warnings + + warnings._deprecated( + "collections.abc.ByteString", + remove=(3, 17), + ) + return super().__new__(cls, name, bases, namespace, **kwargs) + + def __instancecheck__(cls, instance): + import warnings + + warnings._deprecated( + "collections.abc.ByteString", + remove=(3, 17), + ) + return super().__instancecheck__(instance) + +class ByteString(Sequence, metaclass=_DeprecateByteStringMeta): + """Deprecated ABC serving as a common supertype of ``bytes`` and ``bytearray``. + + This ABC is scheduled for removal in Python 3.17. + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the buffer protocol at runtime. For use in type annotations, + either use ``Buffer`` or a union that explicitly specifies the types your + code supports (e.g., ``bytes | bytearray | memoryview``). + """ + + __slots__ = () + +ByteString.register(bytes) +ByteString.register(bytearray) + class MutableSequence(Sequence): """All the operations on a read-write sequence. @@ -1103,15 +1143,16 @@ def reverse(self): self[i], self[n-i-1] = self[n-i-1], self[i] def extend(self, values): - 'S.extend(iterable) -- extend sequence by appending elements from the iterable' + """S.extend(iterable) -- extend sequence by appending elements from the + iterable""" if values is self: values = list(values) for v in values: self.append(v) def pop(self, index=-1): - '''S.pop([index]) -> item -- remove and return item at index (default last). - Raise IndexError if list is empty or index is out of range. + '''S.pop([index]) -> item -- remove and return item at index (default + last). Raise IndexError if list is empty or index is out of range. ''' v = self[index] del self[index] @@ -1130,3 +1171,13 @@ def __iadd__(self, values): MutableSequence.register(list) MutableSequence.register(bytearray) + +_deprecated_ByteString = globals().pop("ByteString") + +def __getattr__(attr): + if attr == "ByteString": + import warnings + warnings._deprecated("collections.abc.ByteString", remove=(3, 17)) + globals()["ByteString"] = _deprecated_ByteString + return _deprecated_ByteString + raise AttributeError(f"module 'collections.abc' has no attribute {attr!r}") diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 4a310a402358b6c..5e0c0124e597b89 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -1,4 +1,4 @@ -import io +import builtins import os import sys @@ -10,13 +10,12 @@ # types if False: - from typing import IO, Self, ClassVar + from typing import IO, Literal, Self, ClassVar _theme: Theme class ANSIColors: RESET = "\x1b[0m" - BLACK = "\x1b[30m" BLUE = "\x1b[34m" CYAN = "\x1b[36m" @@ -75,6 +74,19 @@ class ANSIColors: setattr(NoColors, attr, "") +class CursesColors: + """Curses color constants for terminal UI theming.""" + BLACK = 0 + RED = 1 + GREEN = 2 + YELLOW = 3 + BLUE = 4 + MAGENTA = 5 + CYAN = 6 + WHITE = 7 + DEFAULT = -1 + + # # Experimental theming support (see gh-133346) # @@ -155,7 +167,7 @@ def __iter__(self) -> Iterator[str]: return iter(self.__dataclass_fields__) -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Argparse(ThemeSection): usage: str = ANSIColors.BOLD_BLUE prog: str = ANSIColors.BOLD_MAGENTA @@ -169,13 +181,231 @@ class Argparse(ThemeSection): short_option: str = ANSIColors.BOLD_GREEN label: str = ANSIColors.BOLD_YELLOW action: str = ANSIColors.BOLD_GREEN + default: str = ANSIColors.GREY + interpolated_value: str = ANSIColors.YELLOW + reset: str = ANSIColors.RESET + error: str = ANSIColors.BOLD_MAGENTA + warning: str = ANSIColors.BOLD_YELLOW + message: str = ANSIColors.MAGENTA + + +@dataclass(frozen=True, kw_only=True) +class Ast(ThemeSection): + node: str = ANSIColors.CYAN + field: str = ANSIColors.BLUE + attribute: str = ANSIColors.GREY + string: str = ANSIColors.GREEN + number: str = ANSIColors.YELLOW + keyword: str = ANSIColors.BOLD_BLUE + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class Calendar(ThemeSection): + header: str = ANSIColors.BOLD + highlight: str = ANSIColors.BLACK + ANSIColors.BACKGROUND_YELLOW + weekday: str = ANSIColors.CYAN + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class Difflib(ThemeSection): + """A 'git diff'-like theme for `difflib.unified_diff`.""" + added: str = ANSIColors.GREEN + context: str = ANSIColors.RESET # context lines + header: str = ANSIColors.BOLD # eg "---" and "+++" lines + hunk: str = ANSIColors.CYAN # the "@@" lines + removed: str = ANSIColors.RED + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class FancyCompleter(ThemeSection): + # functions and methods + function: builtins.str = ANSIColors.BOLD_BLUE + builtin_function_or_method: builtins.str = ANSIColors.BOLD_BLUE + method: builtins.str = ANSIColors.BOLD_CYAN + method_wrapper: builtins.str = ANSIColors.BOLD_CYAN + wrapper_descriptor: builtins.str = ANSIColors.BOLD_CYAN + method_descriptor: builtins.str = ANSIColors.BOLD_CYAN + + # numbers + int: builtins.str = ANSIColors.BOLD_YELLOW + float: builtins.str = ANSIColors.BOLD_YELLOW + complex: builtins.str = ANSIColors.BOLD_YELLOW + bool: builtins.str = ANSIColors.BOLD_YELLOW + + # others + type: builtins.str = ANSIColors.BOLD_MAGENTA + module: builtins.str = ANSIColors.CYAN + NoneType: builtins.str = ANSIColors.GREY + bytes: builtins.str = ANSIColors.BOLD_GREEN + str: builtins.str = ANSIColors.BOLD_GREEN + + +@dataclass(frozen=True, kw_only=True) +class HttpServer(ThemeSection): + error: str = ANSIColors.YELLOW + path: str = ANSIColors.CYAN + serving: str = ANSIColors.GREEN + size: str = ANSIColors.GREY + status_informational: str = ANSIColors.RESET + status_ok: str = ANSIColors.GREEN + status_redirect: str = ANSIColors.INTENSE_CYAN + status_client_error: str = ANSIColors.YELLOW + status_server_error: str = ANSIColors.RED + timestamp: str = ANSIColors.GREY + url: str = ANSIColors.CYAN + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class LiveProfiler(ThemeSection): + """Theme section for the live profiling TUI (Tachyon profiler). + + Colors use CursesColors constants (BLACK, RED, GREEN, YELLOW, + BLUE, MAGENTA, CYAN, WHITE, DEFAULT). + """ + # Header colors + title_fg: int = CursesColors.CYAN + title_bg: int = CursesColors.DEFAULT + + # Status display colors + pid_fg: int = CursesColors.CYAN + uptime_fg: int = CursesColors.GREEN + time_fg: int = CursesColors.YELLOW + interval_fg: int = CursesColors.MAGENTA + + # Thread view colors + thread_all_fg: int = CursesColors.GREEN + thread_single_fg: int = CursesColors.MAGENTA + + # Progress bar colors + bar_good_fg: int = CursesColors.GREEN + bar_bad_fg: int = CursesColors.RED + + # Stats colors + on_gil_fg: int = CursesColors.GREEN + off_gil_fg: int = CursesColors.RED + waiting_gil_fg: int = CursesColors.YELLOW + gc_fg: int = CursesColors.MAGENTA + + # Function display colors + func_total_fg: int = CursesColors.CYAN + func_exec_fg: int = CursesColors.GREEN + func_stack_fg: int = CursesColors.YELLOW + func_shown_fg: int = CursesColors.MAGENTA + + # Table header colors (for sorted column highlight) + sorted_header_fg: int = CursesColors.BLACK + sorted_header_bg: int = CursesColors.CYAN + + # Normal header colors (non-sorted columns) - use reverse video style + normal_header_fg: int = CursesColors.BLACK + normal_header_bg: int = CursesColors.WHITE + + # Data row colors + samples_fg: int = CursesColors.CYAN + file_fg: int = CursesColors.GREEN + func_fg: int = CursesColors.YELLOW + + # Trend indicator colors + trend_up_fg: int = CursesColors.GREEN + trend_down_fg: int = CursesColors.RED + + # Medal colors for top functions + medal_gold_fg: int = CursesColors.RED + medal_silver_fg: int = CursesColors.YELLOW + medal_bronze_fg: int = CursesColors.GREEN + + # Background style: 'dark' or 'light' + background_style: Literal["dark", "light"] = "dark" + + +LiveProfilerLight = LiveProfiler( + # Header colors + title_fg=CursesColors.BLUE, # Blue is more readable than cyan on light bg + + # Status display colors - darker colors for light backgrounds + pid_fg=CursesColors.BLUE, + uptime_fg=CursesColors.BLACK, + time_fg=CursesColors.BLACK, + interval_fg=CursesColors.BLUE, + + # Thread view colors + thread_all_fg=CursesColors.BLACK, + thread_single_fg=CursesColors.BLUE, + + # Stats colors + waiting_gil_fg=CursesColors.RED, + gc_fg=CursesColors.BLUE, + + # Function display colors + func_total_fg=CursesColors.BLUE, + func_exec_fg=CursesColors.BLACK, + func_stack_fg=CursesColors.BLACK, + func_shown_fg=CursesColors.BLUE, + + # Table header colors (for sorted column highlight) + sorted_header_fg=CursesColors.WHITE, + sorted_header_bg=CursesColors.BLUE, + + # Normal header colors (non-sorted columns) + normal_header_fg=CursesColors.WHITE, + normal_header_bg=CursesColors.BLACK, + + # Data row colors - use dark colors readable on white + samples_fg=CursesColors.BLACK, + file_fg=CursesColors.BLACK, + func_fg=CursesColors.BLUE, # Blue is more readable than magenta on light bg + + # Medal colors for top functions + medal_silver_fg=CursesColors.BLUE, + + # Background style + background_style="light", +) + + +@dataclass(frozen=True, kw_only=True) +class ProfilerDump(ThemeSection): + header: str = ANSIColors.BOLD_BLUE + interpreter: str = ANSIColors.GREY + thread: str = ANSIColors.BOLD_CYAN + status: str = ANSIColors.YELLOW + frame_index: str = ANSIColors.GREY + frame: str = ANSIColors.BOLD_GREEN + filename: str = ANSIColors.CYAN + line_no: str = ANSIColors.YELLOW + source: str = ANSIColors.WHITE + source_highlight: str = ANSIColors.BOLD_YELLOW + opcode: str = ANSIColors.GREY + warning: str = ANSIColors.YELLOW + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class Pickletools(ThemeSection): + annotation: str = ANSIColors.GREY + arg_number: str = ANSIColors.YELLOW + arg_string: str = ANSIColors.GREEN + mark: str = ANSIColors.GREY + op_call: str = ANSIColors.GREEN + op_container: str = ANSIColors.INTENSE_BLUE + op_memo: str = ANSIColors.MAGENTA + op_meta: str = ANSIColors.GREY + op_stack: str = ANSIColors.BOLD_RED + opcode_code: str = ANSIColors.CYAN + position: str = ANSIColors.GREY + proto: str = ANSIColors.YELLOW reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Syntax(ThemeSection): prompt: str = ANSIColors.BOLD_MAGENTA keyword: str = ANSIColors.BOLD_BLUE + keyword_constant: str = ANSIColors.BOLD_BLUE builtin: str = ANSIColors.CYAN comment: str = ANSIColors.RED string: str = ANSIColors.GREEN @@ -186,10 +416,31 @@ class Syntax(ThemeSection): reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) +class Timeit(ThemeSection): + timing: str = ANSIColors.CYAN + best: str = ANSIColors.BOLD_GREEN + per_loop: str = ANSIColors.GREEN + punctuation: str = ANSIColors.GREY + warning: str = ANSIColors.YELLOW + warning_worst: str = ANSIColors.MAGENTA + warning_best: str = ANSIColors.GREEN + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class Tokenize(ThemeSection): + whitespace: str = ANSIColors.GREY + error: str = ANSIColors.BOLD_RED + position: str = ANSIColors.GREY + delimiter: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) class Traceback(ThemeSection): type: str = ANSIColors.BOLD_MAGENTA message: str = ANSIColors.MAGENTA + note: str = ANSIColors.CYAN filename: str = ANSIColors.MAGENTA line_no: str = ANSIColors.MAGENTA frame: str = ANSIColors.MAGENTA @@ -198,7 +449,7 @@ class Traceback(ThemeSection): reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Unittest(ThemeSection): passed: str = ANSIColors.GREEN warn: str = ANSIColors.YELLOW @@ -207,7 +458,7 @@ class Unittest(ThemeSection): reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Theme: """A suite of themes for all sections of Python. @@ -215,7 +466,17 @@ class Theme: below. """ argparse: Argparse = field(default_factory=Argparse) + ast: Ast = field(default_factory=Ast) + calendar: Calendar = field(default_factory=Calendar) + difflib: Difflib = field(default_factory=Difflib) + fancycompleter: FancyCompleter = field(default_factory=FancyCompleter) + http_server: HttpServer = field(default_factory=HttpServer) + live_profiler: LiveProfiler = field(default_factory=LiveProfiler) + pickletools: Pickletools = field(default_factory=Pickletools) + profiler_dump: ProfilerDump = field(default_factory=ProfilerDump) syntax: Syntax = field(default_factory=Syntax) + timeit: Timeit = field(default_factory=Timeit) + tokenize: Tokenize = field(default_factory=Tokenize) traceback: Traceback = field(default_factory=Traceback) unittest: Unittest = field(default_factory=Unittest) @@ -223,7 +484,17 @@ def copy_with( self, *, argparse: Argparse | None = None, + ast: Ast | None = None, + calendar: Calendar | None = None, + difflib: Difflib | None = None, + fancycompleter: FancyCompleter | None = None, + http_server: HttpServer | None = None, + live_profiler: LiveProfiler | None = None, + pickletools: Pickletools | None = None, + profiler_dump: ProfilerDump | None = None, syntax: Syntax | None = None, + timeit: Timeit | None = None, + tokenize: Tokenize | None = None, traceback: Traceback | None = None, unittest: Unittest | None = None, ) -> Self: @@ -234,7 +505,17 @@ def copy_with( """ return type(self)( argparse=argparse or self.argparse, + ast=ast or self.ast, + calendar=calendar or self.calendar, + difflib=difflib or self.difflib, + fancycompleter=fancycompleter or self.fancycompleter, + http_server=http_server or self.http_server, + live_profiler=live_profiler or self.live_profiler, + pickletools=pickletools or self.pickletools, + profiler_dump=profiler_dump or self.profiler_dump, syntax=syntax or self.syntax, + timeit=timeit or self.timeit, + tokenize=tokenize or self.tokenize, traceback=traceback or self.traceback, unittest=unittest or self.unittest, ) @@ -249,7 +530,17 @@ def no_colors(cls) -> Self: """ return cls( argparse=Argparse.no_colors(), + ast=Ast.no_colors(), + calendar=Calendar.no_colors(), + difflib=Difflib.no_colors(), + fancycompleter=FancyCompleter.no_colors(), + http_server=HttpServer.no_colors(), + live_profiler=LiveProfiler.no_colors(), + pickletools=Pickletools.no_colors(), + profiler_dump=ProfilerDump.no_colors(), syntax=Syntax.no_colors(), + timeit=Timeit.no_colors(), + tokenize=Tokenize.no_colors(), traceback=Traceback.no_colors(), unittest=Unittest.no_colors(), ) @@ -272,21 +563,29 @@ def decolor(text: str) -> str: def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool: + + def _safe_getenv(k: str, fallback: str | None = None) -> str | None: + """Exception-safe environment retrieval. See gh-128636.""" + try: + return os.environ.get(k, fallback) + except Exception: + return fallback + if file is None: file = sys.stdout if not sys.flags.ignore_environment: - if os.environ.get("PYTHON_COLORS") == "0": + if _safe_getenv("PYTHON_COLORS") == "0": return False - if os.environ.get("PYTHON_COLORS") == "1": + if _safe_getenv("PYTHON_COLORS") == "1": return True - if os.environ.get("NO_COLOR"): + if _safe_getenv("NO_COLOR"): return False if not COLORIZE: return False - if os.environ.get("FORCE_COLOR"): + if _safe_getenv("FORCE_COLOR"): return True - if os.environ.get("TERM") == "dumb": + if _safe_getenv("TERM") == "dumb": return False if not hasattr(file, "fileno"): @@ -303,13 +602,16 @@ def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool: try: return os.isatty(file.fileno()) - except io.UnsupportedOperation: + except OSError: return hasattr(file, "isatty") and file.isatty() default_theme = Theme() theme_no_color = default_theme.no_colors() +# Convenience theme with light profiler colors (for white/light terminal backgrounds) +light_profiler_theme = default_theme.copy_with(live_profiler=LiveProfilerLight) + def get_theme( *, @@ -329,7 +631,8 @@ def get_theme( environment (including environment variable state and console configuration on Windows) can also change in the course of the application life cycle. """ - if force_color or (not force_no_color and can_colorize(file=tty_file)): + if force_color or (not force_no_color and + can_colorize(file=tty_file)): return _theme return theme_no_color diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py index a981326432429b3..928db663b446ba2 100644 --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -240,6 +240,7 @@ REVERSE_NAME_MAPPING[('builtins', excname)] = ('exceptions', 'OSError') PYTHON3_IMPORTERROR_EXCEPTIONS = ( + 'ImportCycleError', 'ModuleNotFoundError', ) diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index f168d169a329489..183d0af30acf438 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -2,364 +2,386 @@ # from: # Python/bytecodes.c # Do not edit! -_specializations = { - "RESUME": [ - "RESUME_CHECK", - ], - "TO_BOOL": [ - "TO_BOOL_ALWAYS_TRUE", - "TO_BOOL_BOOL", - "TO_BOOL_INT", - "TO_BOOL_LIST", - "TO_BOOL_NONE", - "TO_BOOL_STR", - ], - "BINARY_OP": [ - "BINARY_OP_MULTIPLY_INT", - "BINARY_OP_ADD_INT", - "BINARY_OP_SUBTRACT_INT", - "BINARY_OP_MULTIPLY_FLOAT", - "BINARY_OP_ADD_FLOAT", - "BINARY_OP_SUBTRACT_FLOAT", - "BINARY_OP_ADD_UNICODE", - "BINARY_OP_SUBSCR_LIST_INT", - "BINARY_OP_SUBSCR_LIST_SLICE", - "BINARY_OP_SUBSCR_TUPLE_INT", - "BINARY_OP_SUBSCR_STR_INT", - "BINARY_OP_SUBSCR_DICT", - "BINARY_OP_SUBSCR_GETITEM", - "BINARY_OP_EXTEND", - "BINARY_OP_INPLACE_ADD_UNICODE", - ], - "STORE_SUBSCR": [ - "STORE_SUBSCR_DICT", - "STORE_SUBSCR_LIST_INT", - ], - "SEND": [ - "SEND_GEN", - ], - "UNPACK_SEQUENCE": [ - "UNPACK_SEQUENCE_TWO_TUPLE", - "UNPACK_SEQUENCE_TUPLE", - "UNPACK_SEQUENCE_LIST", - ], - "STORE_ATTR": [ - "STORE_ATTR_INSTANCE_VALUE", - "STORE_ATTR_SLOT", - "STORE_ATTR_WITH_HINT", - ], - "LOAD_GLOBAL": [ - "LOAD_GLOBAL_MODULE", - "LOAD_GLOBAL_BUILTIN", - ], - "LOAD_SUPER_ATTR": [ - "LOAD_SUPER_ATTR_ATTR", - "LOAD_SUPER_ATTR_METHOD", - ], - "LOAD_ATTR": [ - "LOAD_ATTR_INSTANCE_VALUE", - "LOAD_ATTR_MODULE", - "LOAD_ATTR_WITH_HINT", - "LOAD_ATTR_SLOT", - "LOAD_ATTR_CLASS", - "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK", - "LOAD_ATTR_PROPERTY", - "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - "LOAD_ATTR_METHOD_WITH_VALUES", - "LOAD_ATTR_METHOD_NO_DICT", - "LOAD_ATTR_METHOD_LAZY_DICT", - "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", - "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", - ], - "COMPARE_OP": [ - "COMPARE_OP_FLOAT", - "COMPARE_OP_INT", - "COMPARE_OP_STR", - ], - "CONTAINS_OP": [ - "CONTAINS_OP_SET", - "CONTAINS_OP_DICT", - ], - "JUMP_BACKWARD": [ - "JUMP_BACKWARD_NO_JIT", - "JUMP_BACKWARD_JIT", - ], - "FOR_ITER": [ - "FOR_ITER_LIST", - "FOR_ITER_TUPLE", - "FOR_ITER_RANGE", - "FOR_ITER_GEN", - ], - "CALL": [ - "CALL_BOUND_METHOD_EXACT_ARGS", - "CALL_PY_EXACT_ARGS", - "CALL_TYPE_1", - "CALL_STR_1", - "CALL_TUPLE_1", - "CALL_BUILTIN_CLASS", - "CALL_BUILTIN_O", - "CALL_BUILTIN_FAST", - "CALL_BUILTIN_FAST_WITH_KEYWORDS", - "CALL_LEN", - "CALL_ISINSTANCE", - "CALL_LIST_APPEND", - "CALL_METHOD_DESCRIPTOR_O", - "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - "CALL_METHOD_DESCRIPTOR_NOARGS", - "CALL_METHOD_DESCRIPTOR_FAST", - "CALL_ALLOC_AND_ENTER_INIT", - "CALL_PY_GENERAL", - "CALL_BOUND_METHOD_GENERAL", - "CALL_NON_PY_GENERAL", - ], - "CALL_KW": [ - "CALL_KW_BOUND_METHOD", - "CALL_KW_PY", - "CALL_KW_NON_PY", - ], -} +_specializations = frozendict( + RESUME=( + "RESUME_CHECK", + "RESUME_CHECK_JIT", + ), + TO_BOOL=( + "TO_BOOL_ALWAYS_TRUE", + "TO_BOOL_BOOL", + "TO_BOOL_INT", + "TO_BOOL_LIST", + "TO_BOOL_NONE", + "TO_BOOL_STR", + ), + BINARY_OP=( + "BINARY_OP_MULTIPLY_INT", + "BINARY_OP_ADD_INT", + "BINARY_OP_SUBTRACT_INT", + "BINARY_OP_MULTIPLY_FLOAT", + "BINARY_OP_ADD_FLOAT", + "BINARY_OP_SUBTRACT_FLOAT", + "BINARY_OP_ADD_UNICODE", + "BINARY_OP_SUBSCR_LIST_INT", + "BINARY_OP_SUBSCR_LIST_SLICE", + "BINARY_OP_SUBSCR_TUPLE_INT", + "BINARY_OP_SUBSCR_STR_INT", + "BINARY_OP_SUBSCR_USTR_INT", + "BINARY_OP_SUBSCR_DICT", + "BINARY_OP_SUBSCR_GETITEM", + "BINARY_OP_INPLACE_ADD_UNICODE", + "BINARY_OP_EXTEND", + ), + STORE_SUBSCR=( + "STORE_SUBSCR_DICT", + "STORE_SUBSCR_LIST_INT", + ), + SEND=( + "SEND_GEN", + "SEND_VIRTUAL", + "SEND_ASYNC_GEN", + ), + UNPACK_SEQUENCE=( + "UNPACK_SEQUENCE_TWO_TUPLE", + "UNPACK_SEQUENCE_TUPLE", + "UNPACK_SEQUENCE_LIST", + ), + STORE_ATTR=( + "STORE_ATTR_INSTANCE_VALUE", + "STORE_ATTR_SLOT", + "STORE_ATTR_WITH_HINT", + ), + LOAD_GLOBAL=( + "LOAD_GLOBAL_MODULE", + "LOAD_GLOBAL_BUILTIN", + ), + LOAD_SUPER_ATTR=( + "LOAD_SUPER_ATTR_ATTR", + "LOAD_SUPER_ATTR_METHOD", + ), + LOAD_ATTR=( + "LOAD_ATTR_INSTANCE_VALUE", + "LOAD_ATTR_MODULE", + "LOAD_ATTR_WITH_HINT", + "LOAD_ATTR_SLOT", + "LOAD_ATTR_CLASS", + "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK", + "LOAD_ATTR_PROPERTY", + "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + "LOAD_ATTR_METHOD_WITH_VALUES", + "LOAD_ATTR_METHOD_NO_DICT", + "LOAD_ATTR_METHOD_LAZY_DICT", + "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + ), + COMPARE_OP=( + "COMPARE_OP_FLOAT", + "COMPARE_OP_INT", + "COMPARE_OP_STR", + ), + CONTAINS_OP=( + "CONTAINS_OP_SET", + "CONTAINS_OP_DICT", + ), + JUMP_BACKWARD=( + "JUMP_BACKWARD_NO_JIT", + "JUMP_BACKWARD_JIT", + ), + GET_ITER=( + "GET_ITER_SELF", + "GET_ITER_VIRTUAL", + ), + FOR_ITER=( + "FOR_ITER_LIST", + "FOR_ITER_TUPLE", + "FOR_ITER_RANGE", + "FOR_ITER_GEN", + "FOR_ITER_VIRTUAL", + ), + CALL=( + "CALL_BOUND_METHOD_EXACT_ARGS", + "CALL_PY_EXACT_ARGS", + "CALL_TYPE_1", + "CALL_STR_1", + "CALL_TUPLE_1", + "CALL_BUILTIN_CLASS", + "CALL_BUILTIN_O", + "CALL_BUILTIN_FAST", + "CALL_BUILTIN_FAST_WITH_KEYWORDS", + "CALL_LEN", + "CALL_ISINSTANCE", + "CALL_LIST_APPEND", + "CALL_METHOD_DESCRIPTOR_O", + "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + "CALL_METHOD_DESCRIPTOR_NOARGS", + "CALL_METHOD_DESCRIPTOR_FAST", + "CALL_ALLOC_AND_ENTER_INIT", + "CALL_PY_GENERAL", + "CALL_BOUND_METHOD_GENERAL", + "CALL_NON_PY_GENERAL", + ), + CALL_KW=( + "CALL_KW_BOUND_METHOD", + "CALL_KW_PY", + "CALL_KW_NON_PY", + ), + CALL_FUNCTION_EX=( + "CALL_EX_PY", + "CALL_EX_NON_PY_GENERAL", + ), +) -_specialized_opmap = { - 'BINARY_OP_ADD_FLOAT': 129, - 'BINARY_OP_ADD_INT': 130, - 'BINARY_OP_ADD_UNICODE': 131, - 'BINARY_OP_EXTEND': 132, - 'BINARY_OP_INPLACE_ADD_UNICODE': 3, - 'BINARY_OP_MULTIPLY_FLOAT': 133, - 'BINARY_OP_MULTIPLY_INT': 134, - 'BINARY_OP_SUBSCR_DICT': 135, - 'BINARY_OP_SUBSCR_GETITEM': 136, - 'BINARY_OP_SUBSCR_LIST_INT': 137, - 'BINARY_OP_SUBSCR_LIST_SLICE': 138, - 'BINARY_OP_SUBSCR_STR_INT': 139, - 'BINARY_OP_SUBSCR_TUPLE_INT': 140, - 'BINARY_OP_SUBTRACT_FLOAT': 141, - 'BINARY_OP_SUBTRACT_INT': 142, - 'CALL_ALLOC_AND_ENTER_INIT': 143, - 'CALL_BOUND_METHOD_EXACT_ARGS': 144, - 'CALL_BOUND_METHOD_GENERAL': 145, - 'CALL_BUILTIN_CLASS': 146, - 'CALL_BUILTIN_FAST': 147, - 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 148, - 'CALL_BUILTIN_O': 149, - 'CALL_ISINSTANCE': 150, - 'CALL_KW_BOUND_METHOD': 151, - 'CALL_KW_NON_PY': 152, - 'CALL_KW_PY': 153, - 'CALL_LEN': 154, - 'CALL_LIST_APPEND': 155, - 'CALL_METHOD_DESCRIPTOR_FAST': 156, - 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 157, - 'CALL_METHOD_DESCRIPTOR_NOARGS': 158, - 'CALL_METHOD_DESCRIPTOR_O': 159, - 'CALL_NON_PY_GENERAL': 160, - 'CALL_PY_EXACT_ARGS': 161, - 'CALL_PY_GENERAL': 162, - 'CALL_STR_1': 163, - 'CALL_TUPLE_1': 164, - 'CALL_TYPE_1': 165, - 'COMPARE_OP_FLOAT': 166, - 'COMPARE_OP_INT': 167, - 'COMPARE_OP_STR': 168, - 'CONTAINS_OP_DICT': 169, - 'CONTAINS_OP_SET': 170, - 'FOR_ITER_GEN': 171, - 'FOR_ITER_LIST': 172, - 'FOR_ITER_RANGE': 173, - 'FOR_ITER_TUPLE': 174, - 'JUMP_BACKWARD_JIT': 175, - 'JUMP_BACKWARD_NO_JIT': 176, - 'LOAD_ATTR_CLASS': 177, - 'LOAD_ATTR_CLASS_WITH_METACLASS_CHECK': 178, - 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 179, - 'LOAD_ATTR_INSTANCE_VALUE': 180, - 'LOAD_ATTR_METHOD_LAZY_DICT': 181, - 'LOAD_ATTR_METHOD_NO_DICT': 182, - 'LOAD_ATTR_METHOD_WITH_VALUES': 183, - 'LOAD_ATTR_MODULE': 184, - 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 185, - 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 186, - 'LOAD_ATTR_PROPERTY': 187, - 'LOAD_ATTR_SLOT': 188, - 'LOAD_ATTR_WITH_HINT': 189, - 'LOAD_GLOBAL_BUILTIN': 190, - 'LOAD_GLOBAL_MODULE': 191, - 'LOAD_SUPER_ATTR_ATTR': 192, - 'LOAD_SUPER_ATTR_METHOD': 193, - 'RESUME_CHECK': 194, - 'SEND_GEN': 195, - 'STORE_ATTR_INSTANCE_VALUE': 196, - 'STORE_ATTR_SLOT': 197, - 'STORE_ATTR_WITH_HINT': 198, - 'STORE_SUBSCR_DICT': 199, - 'STORE_SUBSCR_LIST_INT': 200, - 'TO_BOOL_ALWAYS_TRUE': 201, - 'TO_BOOL_BOOL': 202, - 'TO_BOOL_INT': 203, - 'TO_BOOL_LIST': 204, - 'TO_BOOL_NONE': 205, - 'TO_BOOL_STR': 206, - 'UNPACK_SEQUENCE_LIST': 207, - 'UNPACK_SEQUENCE_TUPLE': 208, - 'UNPACK_SEQUENCE_TWO_TUPLE': 209, -} +_specialized_opmap = frozendict( + BINARY_OP_ADD_FLOAT=129, + BINARY_OP_ADD_INT=130, + BINARY_OP_ADD_UNICODE=131, + BINARY_OP_EXTEND=132, + BINARY_OP_INPLACE_ADD_UNICODE=3, + BINARY_OP_MULTIPLY_FLOAT=133, + BINARY_OP_MULTIPLY_INT=134, + BINARY_OP_SUBSCR_DICT=135, + BINARY_OP_SUBSCR_GETITEM=136, + BINARY_OP_SUBSCR_LIST_INT=137, + BINARY_OP_SUBSCR_LIST_SLICE=138, + BINARY_OP_SUBSCR_STR_INT=139, + BINARY_OP_SUBSCR_TUPLE_INT=140, + BINARY_OP_SUBSCR_USTR_INT=141, + BINARY_OP_SUBTRACT_FLOAT=142, + BINARY_OP_SUBTRACT_INT=143, + CALL_ALLOC_AND_ENTER_INIT=144, + CALL_BOUND_METHOD_EXACT_ARGS=145, + CALL_BOUND_METHOD_GENERAL=146, + CALL_BUILTIN_CLASS=147, + CALL_BUILTIN_FAST=148, + CALL_BUILTIN_FAST_WITH_KEYWORDS=149, + CALL_BUILTIN_O=150, + CALL_EX_NON_PY_GENERAL=151, + CALL_EX_PY=152, + CALL_ISINSTANCE=153, + CALL_KW_BOUND_METHOD=154, + CALL_KW_NON_PY=155, + CALL_KW_PY=156, + CALL_LEN=157, + CALL_LIST_APPEND=158, + CALL_METHOD_DESCRIPTOR_FAST=159, + CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS=160, + CALL_METHOD_DESCRIPTOR_NOARGS=161, + CALL_METHOD_DESCRIPTOR_O=162, + CALL_NON_PY_GENERAL=163, + CALL_PY_EXACT_ARGS=164, + CALL_PY_GENERAL=165, + CALL_STR_1=166, + CALL_TUPLE_1=167, + CALL_TYPE_1=168, + COMPARE_OP_FLOAT=169, + COMPARE_OP_INT=170, + COMPARE_OP_STR=171, + CONTAINS_OP_DICT=172, + CONTAINS_OP_SET=173, + FOR_ITER_GEN=174, + FOR_ITER_LIST=175, + FOR_ITER_RANGE=176, + FOR_ITER_TUPLE=177, + FOR_ITER_VIRTUAL=178, + GET_ITER_SELF=179, + GET_ITER_VIRTUAL=180, + JUMP_BACKWARD_JIT=181, + JUMP_BACKWARD_NO_JIT=182, + LOAD_ATTR_CLASS=183, + LOAD_ATTR_CLASS_WITH_METACLASS_CHECK=184, + LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN=185, + LOAD_ATTR_INSTANCE_VALUE=186, + LOAD_ATTR_METHOD_LAZY_DICT=187, + LOAD_ATTR_METHOD_NO_DICT=188, + LOAD_ATTR_METHOD_WITH_VALUES=189, + LOAD_ATTR_MODULE=190, + LOAD_ATTR_NONDESCRIPTOR_NO_DICT=191, + LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES=192, + LOAD_ATTR_PROPERTY=193, + LOAD_ATTR_SLOT=194, + LOAD_ATTR_WITH_HINT=195, + LOAD_GLOBAL_BUILTIN=196, + LOAD_GLOBAL_MODULE=197, + LOAD_SUPER_ATTR_ATTR=198, + LOAD_SUPER_ATTR_METHOD=199, + RESUME_CHECK=200, + RESUME_CHECK_JIT=201, + SEND_ASYNC_GEN=202, + SEND_GEN=203, + SEND_VIRTUAL=204, + STORE_ATTR_INSTANCE_VALUE=205, + STORE_ATTR_SLOT=206, + STORE_ATTR_WITH_HINT=207, + STORE_SUBSCR_DICT=208, + STORE_SUBSCR_LIST_INT=209, + TO_BOOL_ALWAYS_TRUE=210, + TO_BOOL_BOOL=211, + TO_BOOL_INT=212, + TO_BOOL_LIST=213, + TO_BOOL_NONE=214, + TO_BOOL_STR=215, + UNPACK_SEQUENCE_LIST=216, + UNPACK_SEQUENCE_TUPLE=217, + UNPACK_SEQUENCE_TWO_TUPLE=218, +) -opmap = { - 'CACHE': 0, - 'RESERVED': 17, - 'RESUME': 128, - 'INSTRUMENTED_LINE': 254, - 'ENTER_EXECUTOR': 255, - 'BINARY_SLICE': 1, - 'BUILD_TEMPLATE': 2, - 'CALL_FUNCTION_EX': 4, - 'CHECK_EG_MATCH': 5, - 'CHECK_EXC_MATCH': 6, - 'CLEANUP_THROW': 7, - 'DELETE_SUBSCR': 8, - 'END_FOR': 9, - 'END_SEND': 10, - 'EXIT_INIT_CHECK': 11, - 'FORMAT_SIMPLE': 12, - 'FORMAT_WITH_SPEC': 13, - 'GET_AITER': 14, - 'GET_ANEXT': 15, - 'GET_ITER': 16, - 'GET_LEN': 18, - 'GET_YIELD_FROM_ITER': 19, - 'INTERPRETER_EXIT': 20, - 'LOAD_BUILD_CLASS': 21, - 'LOAD_LOCALS': 22, - 'MAKE_FUNCTION': 23, - 'MATCH_KEYS': 24, - 'MATCH_MAPPING': 25, - 'MATCH_SEQUENCE': 26, - 'NOP': 27, - 'NOT_TAKEN': 28, - 'POP_EXCEPT': 29, - 'POP_ITER': 30, - 'POP_TOP': 31, - 'PUSH_EXC_INFO': 32, - 'PUSH_NULL': 33, - 'RETURN_GENERATOR': 34, - 'RETURN_VALUE': 35, - 'SETUP_ANNOTATIONS': 36, - 'STORE_SLICE': 37, - 'STORE_SUBSCR': 38, - 'TO_BOOL': 39, - 'UNARY_INVERT': 40, - 'UNARY_NEGATIVE': 41, - 'UNARY_NOT': 42, - 'WITH_EXCEPT_START': 43, - 'BINARY_OP': 44, - 'BUILD_INTERPOLATION': 45, - 'BUILD_LIST': 46, - 'BUILD_MAP': 47, - 'BUILD_SET': 48, - 'BUILD_SLICE': 49, - 'BUILD_STRING': 50, - 'BUILD_TUPLE': 51, - 'CALL': 52, - 'CALL_INTRINSIC_1': 53, - 'CALL_INTRINSIC_2': 54, - 'CALL_KW': 55, - 'COMPARE_OP': 56, - 'CONTAINS_OP': 57, - 'CONVERT_VALUE': 58, - 'COPY': 59, - 'COPY_FREE_VARS': 60, - 'DELETE_ATTR': 61, - 'DELETE_DEREF': 62, - 'DELETE_FAST': 63, - 'DELETE_GLOBAL': 64, - 'DELETE_NAME': 65, - 'DICT_MERGE': 66, - 'DICT_UPDATE': 67, - 'END_ASYNC_FOR': 68, - 'EXTENDED_ARG': 69, - 'FOR_ITER': 70, - 'GET_AWAITABLE': 71, - 'IMPORT_FROM': 72, - 'IMPORT_NAME': 73, - 'IS_OP': 74, - 'JUMP_BACKWARD': 75, - 'JUMP_BACKWARD_NO_INTERRUPT': 76, - 'JUMP_FORWARD': 77, - 'LIST_APPEND': 78, - 'LIST_EXTEND': 79, - 'LOAD_ATTR': 80, - 'LOAD_COMMON_CONSTANT': 81, - 'LOAD_CONST': 82, - 'LOAD_DEREF': 83, - 'LOAD_FAST': 84, - 'LOAD_FAST_AND_CLEAR': 85, - 'LOAD_FAST_BORROW': 86, - 'LOAD_FAST_BORROW_LOAD_FAST_BORROW': 87, - 'LOAD_FAST_CHECK': 88, - 'LOAD_FAST_LOAD_FAST': 89, - 'LOAD_FROM_DICT_OR_DEREF': 90, - 'LOAD_FROM_DICT_OR_GLOBALS': 91, - 'LOAD_GLOBAL': 92, - 'LOAD_NAME': 93, - 'LOAD_SMALL_INT': 94, - 'LOAD_SPECIAL': 95, - 'LOAD_SUPER_ATTR': 96, - 'MAKE_CELL': 97, - 'MAP_ADD': 98, - 'MATCH_CLASS': 99, - 'POP_JUMP_IF_FALSE': 100, - 'POP_JUMP_IF_NONE': 101, - 'POP_JUMP_IF_NOT_NONE': 102, - 'POP_JUMP_IF_TRUE': 103, - 'RAISE_VARARGS': 104, - 'RERAISE': 105, - 'SEND': 106, - 'SET_ADD': 107, - 'SET_FUNCTION_ATTRIBUTE': 108, - 'SET_UPDATE': 109, - 'STORE_ATTR': 110, - 'STORE_DEREF': 111, - 'STORE_FAST': 112, - 'STORE_FAST_LOAD_FAST': 113, - 'STORE_FAST_STORE_FAST': 114, - 'STORE_GLOBAL': 115, - 'STORE_NAME': 116, - 'SWAP': 117, - 'UNPACK_EX': 118, - 'UNPACK_SEQUENCE': 119, - 'YIELD_VALUE': 120, - 'INSTRUMENTED_END_FOR': 234, - 'INSTRUMENTED_POP_ITER': 235, - 'INSTRUMENTED_END_SEND': 236, - 'INSTRUMENTED_FOR_ITER': 237, - 'INSTRUMENTED_INSTRUCTION': 238, - 'INSTRUMENTED_JUMP_FORWARD': 239, - 'INSTRUMENTED_NOT_TAKEN': 240, - 'INSTRUMENTED_POP_JUMP_IF_TRUE': 241, - 'INSTRUMENTED_POP_JUMP_IF_FALSE': 242, - 'INSTRUMENTED_POP_JUMP_IF_NONE': 243, - 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 244, - 'INSTRUMENTED_RESUME': 245, - 'INSTRUMENTED_RETURN_VALUE': 246, - 'INSTRUMENTED_YIELD_VALUE': 247, - 'INSTRUMENTED_END_ASYNC_FOR': 248, - 'INSTRUMENTED_LOAD_SUPER_ATTR': 249, - 'INSTRUMENTED_CALL': 250, - 'INSTRUMENTED_CALL_KW': 251, - 'INSTRUMENTED_CALL_FUNCTION_EX': 252, - 'INSTRUMENTED_JUMP_BACKWARD': 253, - 'ANNOTATIONS_PLACEHOLDER': 256, - 'JUMP': 257, - 'JUMP_IF_FALSE': 258, - 'JUMP_IF_TRUE': 259, - 'JUMP_NO_INTERRUPT': 260, - 'LOAD_CLOSURE': 261, - 'POP_BLOCK': 262, - 'SETUP_CLEANUP': 263, - 'SETUP_FINALLY': 264, - 'SETUP_WITH': 265, - 'STORE_FAST_MAYBE_NULL': 266, -} +opmap = frozendict( + CACHE=0, + RESERVED=17, + RESUME=128, + INSTRUMENTED_LINE=253, + ENTER_EXECUTOR=254, + TRACE_RECORD=255, + BINARY_SLICE=1, + BUILD_TEMPLATE=2, + CALL_FUNCTION_EX=4, + CHECK_EG_MATCH=5, + CHECK_EXC_MATCH=6, + CLEANUP_THROW=7, + DELETE_SUBSCR=8, + END_FOR=9, + END_SEND=10, + EXIT_INIT_CHECK=11, + FORMAT_SIMPLE=12, + FORMAT_WITH_SPEC=13, + GET_AITER=14, + GET_ANEXT=15, + GET_LEN=16, + INTERPRETER_EXIT=18, + LOAD_BUILD_CLASS=19, + LOAD_LOCALS=20, + MAKE_FUNCTION=21, + MATCH_KEYS=22, + MATCH_MAPPING=23, + MATCH_SEQUENCE=24, + NOP=25, + NOT_TAKEN=26, + POP_EXCEPT=27, + POP_ITER=28, + POP_TOP=29, + PUSH_EXC_INFO=30, + PUSH_NULL=31, + RETURN_GENERATOR=32, + RETURN_VALUE=33, + SETUP_ANNOTATIONS=34, + STORE_SLICE=35, + STORE_SUBSCR=36, + TO_BOOL=37, + UNARY_INVERT=38, + UNARY_NEGATIVE=39, + UNARY_NOT=40, + WITH_EXCEPT_START=41, + BINARY_OP=42, + BUILD_INTERPOLATION=43, + BUILD_LIST=44, + BUILD_MAP=45, + BUILD_SET=46, + BUILD_SLICE=47, + BUILD_STRING=48, + BUILD_TUPLE=49, + CALL=50, + CALL_INTRINSIC_1=51, + CALL_INTRINSIC_2=52, + CALL_KW=53, + COMPARE_OP=54, + CONTAINS_OP=55, + CONVERT_VALUE=56, + COPY=57, + COPY_FREE_VARS=58, + DELETE_ATTR=59, + DELETE_DEREF=60, + DELETE_FAST=61, + DELETE_GLOBAL=62, + DELETE_NAME=63, + DICT_MERGE=64, + DICT_UPDATE=65, + END_ASYNC_FOR=66, + EXTENDED_ARG=67, + FOR_ITER=68, + GET_AWAITABLE=69, + GET_ITER=70, + IMPORT_FROM=71, + IMPORT_NAME=72, + IS_OP=73, + JUMP_BACKWARD=74, + JUMP_BACKWARD_NO_INTERRUPT=75, + JUMP_FORWARD=76, + LIST_APPEND=77, + LIST_EXTEND=78, + LOAD_ATTR=79, + LOAD_COMMON_CONSTANT=80, + LOAD_CONST=81, + LOAD_DEREF=82, + LOAD_FAST=83, + LOAD_FAST_AND_CLEAR=84, + LOAD_FAST_BORROW=85, + LOAD_FAST_BORROW_LOAD_FAST_BORROW=86, + LOAD_FAST_CHECK=87, + LOAD_FAST_LOAD_FAST=88, + LOAD_FROM_DICT_OR_DEREF=89, + LOAD_FROM_DICT_OR_GLOBALS=90, + LOAD_GLOBAL=91, + LOAD_NAME=92, + LOAD_SMALL_INT=93, + LOAD_SPECIAL=94, + LOAD_SUPER_ATTR=95, + MAKE_CELL=96, + MAP_ADD=97, + MATCH_CLASS=98, + POP_JUMP_IF_FALSE=99, + POP_JUMP_IF_NONE=100, + POP_JUMP_IF_NOT_NONE=101, + POP_JUMP_IF_TRUE=102, + RAISE_VARARGS=103, + RERAISE=104, + SEND=105, + SET_ADD=106, + SET_FUNCTION_ATTRIBUTE=107, + SET_UPDATE=108, + STORE_ATTR=109, + STORE_DEREF=110, + STORE_FAST=111, + STORE_FAST_LOAD_FAST=112, + STORE_FAST_STORE_FAST=113, + STORE_GLOBAL=114, + STORE_NAME=115, + SWAP=116, + UNPACK_EX=117, + UNPACK_SEQUENCE=118, + YIELD_VALUE=119, + INSTRUMENTED_END_FOR=233, + INSTRUMENTED_POP_ITER=234, + INSTRUMENTED_END_SEND=235, + INSTRUMENTED_FOR_ITER=236, + INSTRUMENTED_INSTRUCTION=237, + INSTRUMENTED_JUMP_FORWARD=238, + INSTRUMENTED_NOT_TAKEN=239, + INSTRUMENTED_POP_JUMP_IF_TRUE=240, + INSTRUMENTED_POP_JUMP_IF_FALSE=241, + INSTRUMENTED_POP_JUMP_IF_NONE=242, + INSTRUMENTED_POP_JUMP_IF_NOT_NONE=243, + INSTRUMENTED_RESUME=244, + INSTRUMENTED_RETURN_VALUE=245, + INSTRUMENTED_YIELD_VALUE=246, + INSTRUMENTED_END_ASYNC_FOR=247, + INSTRUMENTED_LOAD_SUPER_ATTR=248, + INSTRUMENTED_CALL=249, + INSTRUMENTED_CALL_KW=250, + INSTRUMENTED_CALL_FUNCTION_EX=251, + INSTRUMENTED_JUMP_BACKWARD=252, + ANNOTATIONS_PLACEHOLDER=256, + JUMP=257, + JUMP_IF_FALSE=258, + JUMP_IF_TRUE=259, + JUMP_NO_INTERRUPT=260, + LOAD_CLOSURE=261, + POP_BLOCK=262, + SETUP_CLEANUP=263, + SETUP_FINALLY=264, + SETUP_WITH=265, + STORE_FAST_MAYBE_NULL=266, +) -HAVE_ARGUMENT = 43 -MIN_INSTRUMENTED_OPCODE = 234 +HAVE_ARGUMENT = 41 +MIN_INSTRUMENTED_OPCODE = 233 diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index 0cb064fcd791be0..29b89e311cb1fe7 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -17,7 +17,8 @@ _UNIVERSAL_CONFIG_VARS = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', 'BASECFLAGS', 'BLDSHARED', 'LDSHARED', 'CC', 'CXX', 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS', - 'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS') + 'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS', + 'PY_CORE_EXE_LDFLAGS') # configuration variables that may contain compiler calls _COMPILER_CONFIG_VARS = ('BLDSHARED', 'LDSHARED', 'CC', 'CXX') diff --git a/Lib/_py_warnings.py b/Lib/_py_warnings.py index cbaa94458629ac6..ab09913de6812dd 100644 --- a/Lib/_py_warnings.py +++ b/Lib/_py_warnings.py @@ -369,9 +369,15 @@ def _setoption(arg): if message or module: import re if message: - message = re.escape(message) + if len(message) >= 2 and message[0] == message[-1] == '/': + message = message[1:-1] + else: + message = re.escape(message) if module: - module = re.escape(module) + r'\z' + if len(module) >= 2 and module[0] == module[-1] == '/': + module = module[1:-1] + else: + module = re.escape(module) + r'\z' if lineno: try: lineno = int(lineno) @@ -381,7 +387,23 @@ def _setoption(arg): raise _wm._OptionError("invalid lineno %r" % (lineno,)) from None else: lineno = 0 - _wm.filterwarnings(action, message, category, module, lineno) + try: + _wm.filterwarnings(action, message, category, module, lineno) + except re.PatternError if message or module else (): + if message: + try: + re.compile(message) + except re.PatternError: + raise _wm._OptionError(f"invalid regular expression for " + f"message: {message!r}") from None + if module: + try: + re.compile(module) + except re.PatternError: + raise _wm._OptionError(f"invalid regular expression for " + f"module: {module!r}") from None + # Should never happen. + raise # Helper for _setoption() @@ -449,9 +471,12 @@ def warn(message, category=None, stacklevel=1, source=None, # Check category argument if category is None: category = UserWarning - if not (isinstance(category, type) and issubclass(category, Warning)): - raise TypeError("category must be a Warning subclass, " - "not '{:s}'".format(type(category).__name__)) + elif not isinstance(category, type): + raise TypeError(f"category must be a Warning subclass, not " + f"'{type(category).__name__}'") + elif not issubclass(category, Warning): + raise TypeError(f"category must be a Warning subclass, not " + f"class '{category.__name__}'") if not isinstance(skip_file_prefixes, tuple): # The C version demands a tuple for implementation performance. raise TypeError('skip_file_prefixes must be a tuple of strs.') @@ -495,14 +520,43 @@ def warn(message, category=None, stacklevel=1, source=None, ) +def _match_filename(pattern, filename, *, MS_WINDOWS=(sys.platform == 'win32')): + if not filename: + return pattern.match('') is not None + if filename[0] == '<' and filename[-1] == '>': + return pattern.match(filename) is not None + + is_py = (filename[-3:].lower() == '.py' + if MS_WINDOWS else + filename.endswith('.py')) + if is_py: + filename = filename[:-3] + if pattern.match(filename): # for backward compatibility + return True + if MS_WINDOWS: + if not is_py and filename[-4:].lower() == '.pyw': + filename = filename[:-4] + is_py = True + if is_py and filename[-9:].lower() in (r'\__init__', '/__init__'): + filename = filename[:-9] + filename = filename.replace('\\', '/') + else: + if is_py and filename.endswith('/__init__'): + filename = filename[:-9] + filename = filename.replace('/', '.') + i = 0 + while True: + if pattern.match(filename, i): + return True + i = filename.find('.', i) + 1 + if not i: + return False + + def warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, source=None): lineno = int(lineno) - if module is None: - module = filename or "" - if module[-3:].lower() == ".py": - module = module[:-3] # XXX What about leading pathname? if isinstance(message, Warning): text = str(message) category = message.__class__ @@ -524,9 +578,11 @@ def warn_explicit(message, category, filename, lineno, action, msg, cat, mod, ln = item if ((msg is None or msg.match(text)) and issubclass(category, cat) and - (mod is None or mod.match(module)) and - (ln == 0 or lineno == ln)): - break + (ln == 0 or lineno == ln) and + (mod is None or (_match_filename(mod, filename) + if module is None else + mod.match(module)))): + break else: action = _wm.defaultaction # Early exit actions @@ -564,17 +620,18 @@ def warn_explicit(message, category, filename, lineno, linecache.getlines(filename, module_globals) # Print message and context - msg = _wm.WarningMessage(message, category, filename, lineno, source=source) + msg = _wm.WarningMessage(message, category, filename, lineno, + module=module, source=source) _wm._showwarnmsg(msg) class WarningMessage(object): _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file", - "line", "source") + "line", "source", "module") def __init__(self, message, category, filename, lineno, file=None, - line=None, source=None): + line=None, source=None, module=None): self.message = message self.category = category self.filename = filename @@ -582,12 +639,17 @@ def __init__(self, message, category, filename, lineno, file=None, self.file = file self.line = line self.source = source + self.module = module self._category_name = category.__name__ if category else None def __str__(self): - return ("{message : %r, category : %r, filename : %r, lineno : %s, " - "line : %r}" % (self.message, self._category_name, - self.filename, self.lineno, self.line)) + return ("{message : %r, category : %r, module : %r, " + "filename : %r, lineno : %s, line : %r}" % ( + self.message, self._category_name, self.module, + self.filename, self.lineno, self.line)) + + def __repr__(self): + return f'<{type(self).__qualname__} {self}>' class catch_warnings(object): @@ -644,8 +706,8 @@ def __enter__(self): context = None self._filters = self._module.filters self._module.filters = self._filters[:] - self._showwarning = self._module.showwarning self._showwarnmsg_impl = self._module._showwarnmsg_impl + self._showwarning = self._module.showwarning self._module._filters_mutated_lock_held() if self._record: if _use_context: @@ -653,9 +715,9 @@ def __enter__(self): else: log = [] self._module._showwarnmsg_impl = log.append - # Reset showwarning() to the default implementation to make sure - # that _showwarnmsg() calls _showwarnmsg_impl() - self._module.showwarning = self._module._showwarning_orig + # Reset showwarning() to the default implementation to make sure + # that _showwarnmsg() calls _showwarnmsg_impl() + self._module.showwarning = self._module._showwarning_orig else: log = None if self._filter is not None: @@ -670,8 +732,8 @@ def __exit__(self, *exc_info): self._module._warnings_context.set(self._saved_context) else: self._module.filters = self._filters - self._module.showwarning = self._showwarning self._module._showwarnmsg_impl = self._showwarnmsg_impl + self._module.showwarning = self._showwarning self._module._filters_mutated_lock_held() @@ -762,27 +824,27 @@ def __new__(cls, /, *args, **kwargs): arg.__new__ = staticmethod(__new__) - original_init_subclass = arg.__init_subclass__ - # We need slightly different behavior if __init_subclass__ - # is a bound method (likely if it was implemented in Python) - if isinstance(original_init_subclass, MethodType): - original_init_subclass = original_init_subclass.__func__ + if "__init_subclass__" in arg.__dict__: + # __init_subclass__ is directly present on the decorated class. + # Synthesize a wrapper that calls this method directly. + original_init_subclass = arg.__init_subclass__ + # We need slightly different behavior if __init_subclass__ + # is a bound method (likely if it was implemented in Python). + # Otherwise, it likely means it's a builtin such as + # object's implementation of __init_subclass__. + if isinstance(original_init_subclass, MethodType): + original_init_subclass = original_init_subclass.__func__ @functools.wraps(original_init_subclass) def __init_subclass__(*args, **kwargs): _wm.warn(msg, category=category, stacklevel=stacklevel + 1) return original_init_subclass(*args, **kwargs) - - arg.__init_subclass__ = classmethod(__init_subclass__) - # Or otherwise, which likely means it's a builtin such as - # object's implementation of __init_subclass__. else: - @functools.wraps(original_init_subclass) - def __init_subclass__(*args, **kwargs): + def __init_subclass__(cls, *args, **kwargs): _wm.warn(msg, category=category, stacklevel=stacklevel + 1) - return original_init_subclass(*args, **kwargs) + return super(arg, cls).__init_subclass__(*args, **kwargs) - arg.__init_subclass__ = __init_subclass__ + arg.__init_subclass__ = classmethod(__init_subclass__) arg.__deprecated__ = __new__.__deprecated__ = msg __init_subclass__.__deprecated__ = msg diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index bc35823f70144eb..c1448374402de4a 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -55,7 +55,7 @@ def _days_before_year(year): def _days_in_month(year, month): "year, month -> number of days in that month in that year." - assert 1 <= month <= 12, month + assert 1 <= month <= 12, f"month must be in 1..12, not {month}" if month == 2 and _is_leap(year): return 29 return _DAYS_IN_MONTH[month] @@ -213,17 +213,6 @@ def _need_normalize_century(): _normalize_century = True return _normalize_century -_supports_c99 = None -def _can_support_c99(): - global _supports_c99 - if _supports_c99 is None: - try: - _supports_c99 = ( - _time.strftime("%F", (1900, 1, 1, 0, 0, 0, 0, 1, 0)) == "1900-01-01") - except ValueError: - _supports_c99 = False - return _supports_c99 - # Correctly substitute for %z and %Z escapes in strftime formats. def _wrap_strftime(object, format, timetuple): # Don't call utcoffset() or tzname() unless actually needed. @@ -283,7 +272,7 @@ def _wrap_strftime(object, format, timetuple): newformat.append(Zreplace) # Note that datetime(1000, 1, 1).strftime('%G') == '1000' so # year 1000 for %G can go on the fast path. - elif ((ch in 'YG' or ch in 'FC' and _can_support_c99()) and + elif ((ch in 'YG' or ch in 'FC') and object.year < 1000 and _need_normalize_century()): if ch == 'G': year = int(_time.strftime("%G", timetuple)) @@ -1083,7 +1072,11 @@ def fromisocalendar(cls, year, week, day): @classmethod def strptime(cls, date_string, format): - """Parse a date string according to the given format (like time.strptime()).""" + """Parse string according to the given date format (like time.strptime()). + + For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes + """ import _strptime return _strptime._strptime_datetime_date(cls, date_string, format) @@ -1120,6 +1113,8 @@ def strftime(self, format): Format using strftime(). Example: "%d/%m/%Y, %H:%M:%S" + For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes """ return _wrap_strftime(self, format, self.timetuple()) @@ -1310,7 +1305,7 @@ def __reduce__(self): class tzinfo: - """Abstract base class for time zone info classes. + """Abstract base class for time zone info objects. Subclasses must override the tzname(), utcoffset() and dst() methods. """ @@ -1467,8 +1462,13 @@ def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold return self @classmethod + def strptime(cls, date_string, format): - """string, format -> new time parsed from a string (like time.strptime()).""" + """Parse string according to the given time format (like time.strptime()). + + For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes + """ import _strptime return _strptime._strptime_datetime_time(cls, date_string, format) @@ -1661,6 +1661,9 @@ def fromisoformat(cls, time_string): def strftime(self, format): """Format using strftime(). The date part of the timestamp passed to underlying strftime should not be used. + + For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes """ # The year must be >= 1000 else Python's strftime implementation # can raise a bogus exception. @@ -1776,7 +1779,7 @@ def __reduce__(self): class datetime(date): - """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]]) + """A combination of a date and a time. The year, month and day arguments are required. tzinfo may be None, or an instance of a tzinfo subclass. The remaining arguments may be ints. @@ -1984,7 +1987,7 @@ def fromisoformat(cls, date_string): if became_next_day: year, month, day = date_components # Only wrap day/month when it was previously valid - if month <= 12 and day <= (days_in_month := _days_in_month(year, month)): + if 1 <= month <= 12 and day <= (days_in_month := _days_in_month(year, month)): # Calculate midnight of the next day day += 1 if day > days_in_month: @@ -2209,7 +2212,11 @@ def __str__(self): @classmethod def strptime(cls, date_string, format): - 'string, format -> new datetime parsed from a string (like time.strptime()).' + """Parse string according to the given time format (like time.strptime()). + + For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes + """ import _strptime return _strptime._strptime_datetime_datetime(cls, date_string, format) @@ -2435,6 +2442,8 @@ def _isoweek1monday(year): class timezone(tzinfo): + """Fixed offset from UTC implementation of tzinfo.""" + __slots__ = '_offset', '_name' # Sentinel value to disallow None diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 9b8e42a2342536b..8c0afd14d616e85 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -47,13 +47,16 @@ 'HAVE_THREADS', # C version: compile time choice that enables the coroutine local context - 'HAVE_CONTEXTVAR' + 'HAVE_CONTEXTVAR', + + # Highest version of the spec this module complies with + 'SPEC_VERSION', ] __xname__ = __name__ # sys.modules lookup (--without-threads) __name__ = 'decimal' # For pickling -__version__ = '1.70' # Highest version of the spec this complies with - # See http://speleotrove.com/decimal/ +SPEC_VERSION = '1.70' # Highest version of the spec this complies with + # See https://speleotrove.com/decimal/decarith.html __libmpdec_version__ = "2.4.2" # compatible libmpdec version import math as _math @@ -104,8 +107,8 @@ class DecimalException(ArithmeticError): anything, though. handle -- Called when context._raise_error is called and the - trap_enabler is not set. First argument is self, second is the - context. More arguments can be given, those being after + trap_enabler is not set. First argument is self, second is + the context. More arguments can be given, those being after the explanation in _raise_error (For example, context._raise_error(NewError, '(-x)!', self._sign) would call NewError().handle(context, self._sign).) @@ -222,11 +225,12 @@ class InvalidContext(InvalidOperation): """Invalid context. Unknown rounding, for example. This occurs and signals invalid-operation if an invalid context was - detected during an operation. This can occur if contexts are not checked - on creation and either the precision exceeds the capability of the - underlying concrete representation or an unknown or unsupported rounding - was specified. These aspects of the context need only be checked when - the values are required to be used. The result is [0,qNaN]. + detected during an operation. This can occur if contexts are not + checked on creation and either the precision exceeds the capability of + the underlying concrete representation or an unknown or unsupported + rounding was specified. These aspects of the context need only be + checked when the values are required to be used. The result is + [0,qNaN]. """ def handle(self, context, *args): @@ -319,8 +323,9 @@ class FloatOperation(DecimalException, TypeError): Decimal.from_float() or context.create_decimal_from_float() do not set the flag. - Otherwise (the signal is trapped), only equality comparisons and explicit - conversions are silent. All other mixed operations raise FloatOperation. + Otherwise (the signal is trapped), only equality comparisons and + explicit conversions are silent. All other mixed operations raise + FloatOperation. """ # List of public traps and flags @@ -2898,8 +2903,8 @@ def compare_total(self, other, context=None): """Compares self to other using the abstract representations. This is not like the standard compare, which use their numerical - value. Note that a total ordering is defined for all possible abstract - representations. + value. Note that a total ordering is defined for all possible + abstract representations. """ other = _convert_other(other, raiseit=True) @@ -2970,7 +2975,8 @@ def compare_total(self, other, context=None): def compare_total_mag(self, other, context=None): """Compares self to other using abstract repr., ignoring sign. - Like compare_total, but with operand's sign ignored and assumed to be 0. + Like compare_total, but with operand's sign ignored and assumed to + be 0. """ other = _convert_other(other, raiseit=True) @@ -3340,7 +3346,10 @@ def _fill_logical(self, context, opa, opb): return opa, opb def logical_and(self, other, context=None): - """Applies an 'and' operation between self and other's digits.""" + """Applies an 'and' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -3357,14 +3366,20 @@ def logical_and(self, other, context=None): return _dec_from_triple(0, result.lstrip('0') or '0', 0) def logical_invert(self, context=None): - """Invert all its digits.""" + """Invert all its digits. + + The self must be logical number. + """ if context is None: context = getcontext() return self.logical_xor(_dec_from_triple(0,'1'*context.prec,0), context) def logical_or(self, other, context=None): - """Applies an 'or' operation between self and other's digits.""" + """Applies an 'or' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -3381,7 +3396,10 @@ def logical_or(self, other, context=None): return _dec_from_triple(0, result.lstrip('0') or '0', 0) def logical_xor(self, other, context=None): - """Applies an 'xor' operation between self and other's digits.""" + """Applies an 'xor' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -4095,9 +4113,9 @@ def create_decimal_from_float(self, f): def abs(self, a): """Returns the absolute value of the operand. - If the operand is negative, the result is the same as using the minus - operation on the operand. Otherwise, the result is the same as using - the plus operation on the operand. + If the operand is negative, the result is the same as using the + minus operation on the operand. Otherwise, the result is the same + as using the plus operation on the operand. >>> ExtendedContext.abs(Decimal('2.1')) Decimal('2.1') @@ -4153,16 +4171,17 @@ def canonical(self, a): def compare(self, a, b): """Compares values numerically. - If the signs of the operands differ, a value representing each operand - ('-1' if the operand is less than zero, '0' if the operand is zero or - negative zero, or '1' if the operand is greater than zero) is used in - place of that operand for the comparison instead of the actual - operand. + If the signs of the operands differ, a value representing each + operand ('-1' if the operand is less than zero, '0' if the operand + is zero or negative zero, or '1' if the operand is greater than + zero) is used in place of that operand for the comparison instead of + the actual operand. - The comparison is then effected by subtracting the second operand from - the first and then returning a value according to the result of the - subtraction: '-1' if the result is less than zero, '0' if the result is - zero or negative zero, or '1' if the result is greater than zero. + The comparison is then effected by subtracting the second operand + from the first and then returning a value according to the result of + the subtraction: '-1' if the result is less than zero, '0' if the + result is zero or negative zero, or '1' if the result is greater + than zero. >>> ExtendedContext.compare(Decimal('2.1'), Decimal('3')) Decimal('-1') @@ -4225,8 +4244,8 @@ def compare_total(self, a, b): """Compares two operands using their abstract representation. This is not like the standard compare, which use their numerical - value. Note that a total ordering is defined for all possible abstract - representations. + value. Note that a total ordering is defined for all possible + abstract representations. >>> ExtendedContext.compare_total(Decimal('12.73'), Decimal('127.9')) Decimal('-1') @@ -4253,7 +4272,8 @@ def compare_total(self, a, b): def compare_total_mag(self, a, b): """Compares two operands using their abstract representation ignoring sign. - Like compare_total, but with operand's sign ignored and assumed to be 0. + Like compare_total, but with operand's sign ignored and assumed to + be 0. """ a = _convert_other(a, raiseit=True) return a.compare_total_mag(b) @@ -4911,8 +4931,8 @@ def multiply(self, a, b): If either operand is a special value then the general rules apply. Otherwise, the operands are multiplied together - ('long multiplication'), resulting in a number which may be as long as - the sum of the lengths of the two operands. + ('long multiplication'), resulting in a number which may be as long + as the sum of the lengths of the two operands. >>> ExtendedContext.multiply(Decimal('1.20'), Decimal('3')) Decimal('3.60') @@ -5188,19 +5208,19 @@ def quantize(self, a, b): """Returns a value equal to 'a' (rounded), having the exponent of 'b'. The coefficient of the result is derived from that of the left-hand - operand. It may be rounded using the current rounding setting (if the - exponent is being increased), multiplied by a positive power of ten (if - the exponent is being decreased), or is unchanged (if the exponent is - already equal to that of the right-hand operand). + operand. It may be rounded using the current rounding setting (if + the exponent is being increased), multiplied by a positive power of + ten (if the exponent is being decreased), or is unchanged (if the + exponent is already equal to that of the right-hand operand). Unlike other operations, if the length of the coefficient after the quantize operation would be greater than precision then an Invalid - operation condition is raised. This guarantees that, unless there is - an error condition, the exponent of the result of a quantize is always - equal to that of the right-hand operand. + operation condition is raised. This guarantees that, unless there + is an error condition, the exponent of the result of a quantize is + always equal to that of the right-hand operand. - Also unlike other operations, quantize will never raise Underflow, even - if the result is subnormal and inexact. + Also unlike other operations, quantize will never raise Underflow, + even if the result is subnormal and inexact. >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.001')) Decimal('2.170') @@ -5254,13 +5274,13 @@ def remainder(self, a, b): """Returns the remainder from integer division. The result is the residue of the dividend after the operation of - calculating integer division as described for divide-integer, rounded - to precision digits if necessary. The sign of the result, if - non-zero, is the same as that of the original dividend. + calculating integer division as described for divide-integer, + rounded to precision digits if necessary. The sign of the result, + if non-zero, is the same as that of the original dividend. - This operation will fail under the same conditions as integer division - (that is, if integer division on the same two operands would fail, the - remainder cannot be calculated). + This operation will fail under the same conditions as integer + division (that is, if integer division on the same two operands + would fail, the remainder cannot be calculated). >>> ExtendedContext.remainder(Decimal('2.1'), Decimal('3')) Decimal('2.1') @@ -5294,9 +5314,9 @@ def remainder_near(self, a, b): is chosen). If the result is equal to 0 then its sign will be the sign of a. - This operation will fail under the same conditions as integer division - (that is, if integer division on the same two operands would fail, the - remainder cannot be calculated). + This operation will fail under the same conditions as integer + division (that is, if integer division on the same two operands + would fail, the remainder cannot be calculated). >>> ExtendedContext.remainder_near(Decimal('2.1'), Decimal('3')) Decimal('-0.9') @@ -5354,8 +5374,8 @@ def rotate(self, a, b): def same_quantum(self, a, b): """Returns True if the two operands have the same exponent. - The result is never affected by either the sign or the coefficient of - either operand. + The result is never affected by either the sign or the coefficient + of either operand. >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.001')) False @@ -5427,8 +5447,8 @@ def shift(self, a, b): def sqrt(self, a): """Square root of a non-negative number to context precision. - If the result must be inexact, it is rounded using the round-half-even - algorithm. + If the result must be inexact, it is rounded using the + round-half-even algorithm. >>> ExtendedContext.sqrt(Decimal('0')) Decimal('0') @@ -6387,3 +6407,11 @@ def _format_number(is_negative, intpart, fracpart, exp, spec): # _PyHASH_10INV is the inverse of 10 modulo the prime _PyHASH_MODULUS _PyHASH_10INV = pow(10, _PyHASH_MODULUS - 2, _PyHASH_MODULUS) del sys + +def __getattr__(name): + if name == "__version__": + from warnings import _deprecated + + _deprecated("__version__", remove=(3, 20)) + return SPEC_VERSION + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 5db8ce9244b5ba6..4ba9b4070dff93e 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -10,7 +10,7 @@ import sys # Import _thread instead of threading to reduce startup cost from _thread import allocate_lock as Lock -if sys.platform in {'win32', 'cygwin'}: +if sys.platform == 'win32': from msvcrt import setmode as _setmode else: _setmode = None @@ -83,27 +83,28 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, wrapped. (If a file descriptor is given, it is closed when the returned I/O object is closed, unless closefd is set to False.) - mode is an optional string that specifies the mode in which the file is - opened. It defaults to 'r' which means open for reading in text mode. Other - common values are 'w' for writing (truncating the file if it already - exists), 'x' for exclusive creation of a new file, and 'a' for appending - (which on some Unix systems, means that all writes append to the end of the - file regardless of the current seek position). In text mode, if encoding is - not specified the encoding used is platform dependent. (For reading and - writing raw bytes use binary mode and leave encoding unspecified.) The - available modes are: - - ========= =============================================================== + mode is an optional string that specifies the mode in which the file + is opened. It defaults to 'r' which means open for reading in text + mode. Other common values are 'w' for writing (truncating the file if + it already exists), 'x' for exclusive creation of a new file, and + 'a' for appending (which on some Unix systems, means that all writes + append to the end of the file regardless of the current seek position). + In text mode, if encoding is not specified the encoding used is platform + dependent. (For reading and writing raw bytes use binary mode and leave + encoding unspecified.) The available modes are: + + ========= ========================================================== Character Meaning - --------- --------------------------------------------------------------- + --------- ---------------------------------------------------------- 'r' open for reading (default) 'w' open for writing, truncating the file first 'x' create a new file and open it for writing - 'a' open for writing, appending to the end of the file if it exists + 'a' open for writing, appending to the end of the file if it + exists 'b' binary mode 't' text mode (default) '+' open a disk file for updating (reading and writing) - ========= =============================================================== + ========= ========================================================== The default mode is 'rt' (open for reading text). For binary random access, the mode 'w+b' opens and truncates the file to 0 bytes, while @@ -111,23 +112,23 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, raises an `FileExistsError` if the file already exists. Python distinguishes between files opened in binary and text modes, - even when the underlying operating system doesn't. Files opened in + even when the underlying operating system doesn't. Files opened in binary mode (appending 'b' to the mode argument) return contents as - bytes objects without any decoding. In text mode (the default, or when + bytes objects without any decoding. In text mode (the default, or when 't' is appended to the mode argument), the contents of the file are returned as strings, the bytes having been first decoded using a platform-dependent encoding or using the specified encoding if given. buffering is an optional integer used to set the buffering policy. - Pass 0 to switch buffering off (only allowed in binary mode), 1 to select - line buffering (only usable in text mode), and an integer > 1 to indicate - the size of a fixed-size chunk buffer. When no buffering argument is - given, the default buffering policy works as follows: + Pass 0 to switch buffering off (only allowed in binary mode), 1 to + select line buffering (only usable in text mode), and an integer > 1 to + indicate the size of a fixed-size chunk buffer. When no buffering + argument is given, the default buffering policy works as follows: - * Binary files are buffered in fixed-size chunks; the size of the buffer - is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) - when the device block size is available. - On most systems, the buffer will typically be 128 kilobytes long. + * Binary files are buffered in fixed-size chunks; the size of the buffer + is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) when the device + block size is available. + On most systems, the buffer will typically be 128 kilobytes long. * "Interactive" text files (files for which isatty() returns True) use line buffering. Other text files use the policy described above @@ -147,8 +148,8 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, encoding error strings. newline is a string controlling how universal newlines works (it only - applies to text mode). It can be None, '', '\n', '\r', and '\r\n'. It works - as follows: + applies to text mode). It can be None, '', '\n', '\r', and '\r\n'. It + works as follows: * On input, if newline is None, universal newlines mode is enabled. Lines in the input can end in '\n', '\r', or '\r\n', and @@ -164,17 +165,17 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, other legal values, any '\n' characters written are translated to the given string. - closedfd is a bool. If closefd is False, the underlying file descriptor will - be kept open when the file is closed. This does not work when a file name is - given and must be True in that case. + closedfd is a bool. If closefd is False, the underlying file descriptor + will be kept open when the file is closed. This does not work when + a file name is given and must be True in that case. The newly created file is non-inheritable. A custom opener can be used by passing a callable as *opener*. The - underlying file descriptor for the file object is then obtained by calling - *opener* with (*file*, *flags*). *opener* must return an open file - descriptor (passing os.open as *opener* results in functionality similar to - passing None). + underlying file descriptor for the file object is then obtained by + calling *opener* with (*file*, *flags*). *opener* must return an open + file descriptor (passing os.open as *opener* results in functionality + similar to passing None). open() returns a file object whose type depends on the mode, and through which the standard file operations such as reading and writing @@ -351,10 +352,12 @@ def seek(self, pos, whence=0): interpreted relative to the position indicated by whence. Values for whence are ints: - * 0 -- start of stream (the default); offset should be zero or positive + * 0 -- start of stream (the default); offset should be zero or + positive * 1 -- current stream position; offset may be negative * 2 -- end of stream; offset is usually negative - Some operating systems / file systems could provide additional values. + Some operating systems / file systems could provide additional + values. Return an int indicating the new absolute position. """ @@ -367,8 +370,8 @@ def tell(self): def truncate(self, pos=None): """Truncate file to size bytes. - Size defaults to the current IO position as reported by tell(). Return - the new size. + Size defaults to the current IO position as reported by tell(). + Return the new size. """ self._unsupported("truncate") @@ -492,7 +495,8 @@ def __exit__(self, *args): def fileno(self): """Returns underlying file descriptor (an int) if one exists. - An OSError is raised if the IO object does not use a file descriptor. + An OSError is raised if the IO object does not use a file + descriptor. """ self._unsupported("fileno") @@ -546,7 +550,7 @@ def nreadahead(): res += b if res.endswith(b"\n"): break - return bytes(res) + return res.take_bytes() def __iter__(self): self._checkClosed() @@ -617,8 +621,10 @@ def read(self, size=-1): n = self.readinto(b) if n is None: return None + if n < 0 or n > len(b): + raise ValueError(f"readinto returned {n} outside buffer size {len(b)}") del b[n:] - return bytes(b) + return b.take_bytes() def readall(self): """Read until EOF, using multiple read() call.""" @@ -626,7 +632,7 @@ def readall(self): while data := self.read(DEFAULT_BUFFER_SIZE): res += data if res: - return bytes(res) + return res.take_bytes() else: # b'' or None return data @@ -939,7 +945,7 @@ def read(self, size=-1): newpos = min(len(self._buffer), self._pos + size) b = self._buffer[self._pos : newpos] self._pos = newpos - return bytes(b) + return b.take_bytes() def read1(self, size=-1): """This is the same as read. @@ -947,23 +953,24 @@ def read1(self, size=-1): return self.read(size) def write(self, b): - if self.closed: - raise ValueError("write to closed file") if isinstance(b, str): raise TypeError("can't write str to binary stream") with memoryview(b) as view: + if self.closed: + raise ValueError("write to closed file") + n = view.nbytes # Size of any bytes-like object - if n == 0: - return 0 + if n == 0: + return 0 - with self._lock: - pos = self._pos - if pos > len(self._buffer): - # Pad buffer to pos with null bytes. - self._buffer.resize(pos) - self._buffer[pos:pos + n] = b - self._pos += n - return n + with self._lock: + pos = self._pos + if pos > len(self._buffer): + # Pad buffer to pos with null bytes. + self._buffer.resize(pos) + self._buffer[pos:pos + n] = view + self._pos += n + return n def seek(self, pos, whence=0): if self.closed: @@ -1498,20 +1505,26 @@ class FileIO(RawIOBase): _writable = False _appending = False _seekable = None + _truncate = False _closefd = True def __init__(self, file, mode='r', closefd=True, opener=None): - """Open a file. The mode can be 'r' (default), 'w', 'x' or 'a' for reading, - writing, exclusive creation or appending. The file will be created if it - doesn't exist when opened for writing or appending; it will be truncated - when opened for writing. A FileExistsError will be raised if it already - exists when opened for creating. Opening a file for creating implies - writing so this mode behaves in a similar way to 'w'. Add a '+' to the mode - to allow simultaneous reading and writing. A custom opener can be used by - passing a callable as *opener*. The underlying file descriptor for the file - object is then obtained by calling opener with (*name*, *flags*). - *opener* must return an open file descriptor (passing os.open as *opener* - results in functionality similar to passing None). + """Open a file. + + The mode can be 'r' (default), 'w', 'x' or 'a' for reading, + writing, exclusive creation or appending. The file will be created + if it doesn't exist when opened for writing or appending; it will be + truncated when opened for writing. A FileExistsError will be raised + if it already exists when opened for creating. Opening a file for + creating implies writing so this mode behaves in a similar way to + 'w'. Add a '+' to the mode to allow simultaneous reading and + writing. + + A custom opener can be used by passing a callable as *opener*. + The underlying file descriptor for the file object is then obtained + by calling opener with (*name*, *flags*). *opener* must return + an open file descriptor (passing os.open as *opener* results in + functionality similar to passing None). """ if self._fd >= 0: # Have to close the existing file first. @@ -1553,6 +1566,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): flags = 0 elif 'w' in mode: self._writable = True + self._truncate = True flags = os.O_CREAT | os.O_TRUNC elif 'a' in mode: self._writable = True @@ -1734,7 +1748,7 @@ def readall(self): assert len(result) - bytes_read >= 1, \ "os.readinto buffer size 0 will result in erroneous EOF / returns 0" result.resize(bytes_read) - return bytes(result) + return result.take_bytes() def readinto(self, buffer): """Same as RawIOBase.readinto().""" @@ -1749,8 +1763,8 @@ def write(self, b): """Write bytes b to file, return number written. Only makes one system call, so not all of the data may be written. - The number of bytes actually written is returned. In non-blocking mode, - returns None if the write would block. + The number of bytes actually written is returned. In non-blocking + mode, returns None if the write would block. """ self._checkClosed() self._checkWritable() @@ -1762,11 +1776,12 @@ def write(self, b): def seek(self, pos, whence=SEEK_SET): """Move to new file position. - Argument offset is a byte count. Optional argument whence defaults to - SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values - are SEEK_CUR or 1 (move relative to current position, positive or negative), - and SEEK_END or 2 (move relative to end of file, usually negative, although - many platforms allow seeking beyond the end of a file). + Argument offset is a byte count. Optional argument whence defaults + to SEEK_SET or 0 (offset from start of file, offset should be >= 0); + other values are SEEK_CUR or 1 (move relative to current position, + positive or negative), and SEEK_END or 2 (move relative to end of + file, usually negative, although many platforms allow seeking beyond + the end of a file). Note that not all file objects are seekable. """ @@ -1799,8 +1814,8 @@ def truncate(self, size=None): def close(self): """Close the file. - A closed file cannot be used for further I/O operations. close() may be - called more than once without error. + A closed file cannot be used for further I/O operations. + close() may be called more than once without error. """ if not self.closed: self._stat_atopen = None @@ -1877,7 +1892,10 @@ def mode(self): return 'ab' elif self._readable: if self._writable: - return 'rb+' + if self._truncate: + return 'wb+' + else: + return 'rb+' else: return 'rb' else: @@ -1895,8 +1913,8 @@ class TextIOBase(IOBase): def read(self, size=-1): """Read at most size characters from stream, where size is an int. - Read from underlying buffer until we have size characters or we hit EOF. - If size is negative or omitted, read until EOF. + Read from underlying buffer until we have size characters or we hit + EOF. If size is negative or omitted, read until EOF. Returns a string. """ @@ -1910,7 +1928,7 @@ def truncate(self, pos=None): """Truncate size to pos, where pos is an int.""" self._unsupported("truncate") - def readline(self): + def readline(self, size=-1, /): """Read until newline or EOF. Returns an empty string if EOF is hit immediately. diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index 1e9462a42156d45..17bf5cdc819542d 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -1,19 +1,47 @@ from __future__ import annotations +import importlib +import os import pkgutil +import re import sys import token import tokenize +from importlib.machinery import FileFinder from io import StringIO from contextlib import contextmanager from dataclasses import dataclass from itertools import chain from tokenize import TokenInfo +from .fancycompleter import safe_getattr TYPE_CHECKING = False if TYPE_CHECKING: + from types import ModuleType from typing import Any, Iterable, Iterator, Mapping + from .types import CompletionAction + + +HARDCODED_SUBMODULES = { + # Standard library submodules that are not detected by pkgutil.iter_modules + # but can be imported, so should be proposed in completion + "collections": ["abc"], + "math": ["integer"], + "os": ["path"], + "xml.parsers.expat": ["errors", "model"], +} + +AUTO_IMPORT_DENYLIST = { + # Standard library modules/submodules that have import side effects + # and must not be automatically imported to complete attributes + re.compile(r"antigravity"), # Calls webbrowser.open + re.compile(r"idlelib\..+"), # May open IDLE GUI + re.compile(r"test\..+"), # Various side-effects + re.compile(r"this"), # Prints to stdout + re.compile(r"_ios_support"), # Spawns a subprocess + re.compile(r".+\.__main__"), # Should not be imported +} def make_default_module_completer() -> ModuleCompleter: @@ -40,36 +68,73 @@ class ModuleCompleter: def __init__(self, namespace: Mapping[str, Any] | None = None) -> None: self.namespace = namespace or {} self._global_cache: list[pkgutil.ModuleInfo] = [] + self._failed_imports: set[str] = set() self._curr_sys_path: list[str] = sys.path[:] + self._stdlib_path = os.path.dirname(importlib.__path__[0]) - def get_completions(self, line: str) -> list[str] | None: - """Return the next possible import completions for 'line'.""" + def get_completions( + self, line: str, *, include_values: bool = True + ) -> tuple[list[str], list[Any], CompletionAction | None] | None: + """Return the next possible import completions for 'line'. + + For attributes completion, if the module to complete from is not + imported, also return an action (prompt + callback to run if the + user press TAB again) to import the module. + + If *include_values* is false, the returned values list is empty and + attribute values are not resolved. + """ result = ImportParser(line).parse() if not result: return None try: - return self.complete(*result) + return self.complete(*result, include_values=include_values) except Exception: # Some unexpected error occurred, make it look like # no completions are available - return [] - - def complete(self, from_name: str | None, name: str | None) -> list[str]: + return [], [], None + + def complete( + self, + from_name: str | None, + name: str | None, + *, + include_values: bool = True, + ) -> tuple[list[str], list[Any], CompletionAction | None]: if from_name is None: # import x.y.z assert name is not None path, prefix = self.get_path_and_prefix(name) modules = self.find_modules(path, prefix) - return [self.format_completion(path, module) for module in modules] + names = [self.format_completion(path, module) for module in modules] + # These are always modules, use dummy values to get the right color + values = [sys] * len(names) if include_values else [] + return names, values, None if name is None: # from x.y.z path, prefix = self.get_path_and_prefix(from_name) modules = self.find_modules(path, prefix) - return [self.format_completion(path, module) for module in modules] + names = [self.format_completion(path, module) for module in modules] + # These are always modules, use dummy values to get the right color + values = [sys] * len(names) if include_values else [] + return names, values, None # from x.y import z - return self.find_modules(from_name, name) + submodules = self.find_modules(from_name, name) + attr_names, attr_module, action = self._find_attributes(from_name, name) + all_names = sorted({*submodules, *attr_names}) + if not include_values: + return all_names, [], action + + # Build values list matching the sorted order: + # submodules use `sys` as a dummy value so they get the 'module' color, + # attributes use their actual value. + attr_map = {} + if attr_module is not None: + attr_map = {n: safe_getattr(attr_module, n) for n in attr_names} + all_values = [attr_map[n] if n in attr_map else sys for n in all_names] + return all_names, all_values, action def find_modules(self, path: str, prefix: str) -> list[str]: """Find all modules under 'path' that start with 'prefix'.""" @@ -87,20 +152,86 @@ def _find_modules(self, path: str, prefix: str) -> list[str]: if self.is_suggestion_match(module.name, prefix)] return sorted(builtin_modules + third_party_modules) - if path.startswith('.'): - # Convert relative path to absolute path - package = self.namespace.get('__package__', '') - path = self.resolve_relative_name(path, package) # type: ignore[assignment] - if path is None: - return [] + path = self._resolve_relative_path(path) # type: ignore[assignment] + if path is None: + return [] modules: Iterable[pkgutil.ModuleInfo] = self.global_cache + imported_module = sys.modules.get(path.split('.')[0]) + if imported_module: + # Filter modules to those whose name and specs match the + # imported module to avoid invalid suggestions + spec = imported_module.__spec__ + if spec: + def _safe_find_spec(mod: pkgutil.ModuleInfo) -> bool: + try: + return mod.module_finder.find_spec(mod.name, None) == spec + except Exception: + return False + modules = [mod for mod in modules + if mod.name == spec.name + and _safe_find_spec(mod)] + else: + modules = [] + + is_stdlib_import: bool | None = None for segment in path.split('.'): modules = [mod_info for mod_info in modules if mod_info.ispkg and mod_info.name == segment] + if is_stdlib_import is None: + # Top-level import decide if we import from stdlib or not + is_stdlib_import = all( + self._is_stdlib_module(mod_info) for mod_info in modules + ) modules = self.iter_submodules(modules) - return [module.name for module in modules - if self.is_suggestion_match(module.name, prefix)] + + module_names = [module.name for module in modules] + if is_stdlib_import: + module_names.extend(HARDCODED_SUBMODULES.get(path, ())) + return [module_name for module_name in module_names + if self.is_suggestion_match(module_name, prefix)] + + def _is_stdlib_module(self, module_info: pkgutil.ModuleInfo) -> bool: + return (isinstance(module_info.module_finder, FileFinder) + and module_info.module_finder.path == self._stdlib_path) + + def find_attributes( + self, path: str, prefix: str + ) -> tuple[list[str], list[Any], CompletionAction | None]: + """Find all attributes of module 'path' that start with 'prefix'.""" + attributes, module, action = self._find_attributes(path, prefix) + if module is not None: + values = [safe_getattr(module, attr) for attr in attributes] + else: + values = [] + return attributes, values, action + + def _find_attributes( + self, path: str, prefix: str + ) -> tuple[list[str], ModuleType | None, CompletionAction | None]: + path = self._resolve_relative_path(path) # type: ignore[assignment] + if path is None: + return [], None, None + + imported_module = sys.modules.get(path) + if not imported_module: + if path in self._failed_imports: # Do not propose to import again + return [], None, None + imported_module = self._maybe_import_module(path) + if not imported_module: + return [], None, self._get_import_completion_action(path) + try: + module_attributes = dir(imported_module) + except Exception: + module_attributes = [] + # Filter out invalid attribute names, such as dashes that cannot be + # imported with 'import'. + names = [ + attr_name for attr_name in module_attributes + if (self.is_suggestion_match(attr_name, prefix) + and attr_name.isidentifier()) + ] + return names, imported_module, None def is_suggestion_match(self, module_name: str, prefix: str) -> bool: if prefix: @@ -146,6 +277,13 @@ def format_completion(self, path: str, module: str) -> str: return f'{path}{module}' return f'{path}.{module}' + def _resolve_relative_path(self, path: str) -> str | None: + """Resolve a relative import path to absolute. Returns None if unresolvable.""" + if path.startswith('.'): + package = self.namespace.get('__package__', '') + return self.resolve_relative_name(path, package) + return path + def resolve_relative_name(self, name: str, package: str) -> str | None: """Resolve a relative module name to an absolute name. @@ -169,10 +307,40 @@ def global_cache(self) -> list[pkgutil.ModuleInfo]: """Global module cache""" if not self._global_cache or self._curr_sys_path != sys.path: self._curr_sys_path = sys.path[:] - # print('getting packages') self._global_cache = list(pkgutil.iter_modules()) + self._failed_imports.clear() # retry on sys.path change return self._global_cache + def _maybe_import_module(self, fqname: str) -> ModuleType | None: + if any(pattern.fullmatch(fqname) for pattern in AUTO_IMPORT_DENYLIST): + # Special-cased modules with known import side-effects + return None + root = fqname.split(".")[0] + mod_info = next((m for m in self.global_cache if m.name == root), None) + if not mod_info or not self._is_stdlib_module(mod_info): + # Only import stdlib modules (no risk of import side-effects) + return None + try: + return importlib.import_module(fqname) + except Exception: + sys.modules.pop(fqname, None) # Clean half-imported module + return None + + def _get_import_completion_action(self, path: str) -> CompletionAction: + prompt = ("[ module not imported, press again to import it " + "and propose attributes ]") + + def _do_import() -> str | None: + try: + importlib.import_module(path) + return None + except Exception as exc: + sys.modules.pop(path, None) # Clean half-imported module + self._failed_imports.add(path) + return f"[ error during import: {exc} ]" + + return (prompt, _do_import) + class ImportParser: """ diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 50c824995d85b8b..e79fbfa6bb0b386 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -22,6 +22,7 @@ from __future__ import annotations import os import time +from typing import TYPE_CHECKING # Categories of actions: # killing @@ -32,10 +33,11 @@ # finishing # [completion] +from .render import RenderedScreen from .trace import trace # types -if False: +if TYPE_CHECKING: from .historical_reader import HistoricalReader @@ -74,7 +76,7 @@ def kill_range(self, start: int, end: int) -> None: else: r.kill_ring.append(text) r.pos = start - r.dirty = True + r.invalidate_buffer(start) class YankCommand(Command): @@ -125,24 +127,27 @@ def do(self) -> None: r.arg = 10 * r.arg - d else: r.arg = 10 * r.arg + d - r.dirty = True + r.invalidate_prompt() class clear_screen(Command): def do(self) -> None: r = self.reader + trace("command.clear_screen") r.console.clear() - r.dirty = True + r.invalidate_full() class refresh(Command): def do(self) -> None: - self.reader.dirty = True + trace("command.refresh") + self.reader.invalidate_full() class repaint(Command): def do(self) -> None: - self.reader.dirty = True + trace("command.repaint") + self.reader.invalidate_full() self.reader.console.repaint() @@ -208,9 +213,10 @@ def do(self) -> None: repl = len(r.kill_ring[-1]) r.kill_ring.insert(0, r.kill_ring.pop()) t = r.kill_ring[-1] + start = r.pos - repl b[r.pos - repl : r.pos] = t r.pos = r.pos - repl + len(t) - r.dirty = True + r.invalidate_buffer(start) class interrupt(FinishCommand): @@ -242,8 +248,9 @@ def do(self) -> None: r.console.prepare() r.pos = p # r.posxy = 0, 0 # XXX this is invalid - r.dirty = True - r.console.screen = [] + r.invalidate_full() + trace("command.suspend sync_rendered_screen") + r.console.sync_rendered_screen(RenderedScreen.empty(), r.console.posxy) class up(MotionCommand): @@ -369,6 +376,7 @@ class self_insert(EditCommand): def do(self) -> None: r = self.reader text = self.event * r.get_arg() + start = r.pos r.insert(text) if r.paste_mode: data = "" @@ -376,7 +384,7 @@ def do(self) -> None: data += ev.data if data: r.insert(data) - r.last_refresh_cache.invalidated = True + r.invalidate_buffer(start) class insert_nl(EditCommand): @@ -400,40 +408,49 @@ def do(self) -> None: del b[s] b.insert(t, c) r.pos = t - r.dirty = True + r.invalidate_buffer(s) class backspace(EditCommand): def do(self) -> None: r = self.reader b = r.buffer + changed_from: int | None = None for i in range(r.get_arg()): if r.pos > 0: r.pos -= 1 del b[r.pos] - r.dirty = True + changed_from = r.pos if changed_from is None else min(changed_from, r.pos) else: self.reader.error("can't backspace at start") + if changed_from is not None: + r.invalidate_buffer(changed_from) class delete(EditCommand): def do(self) -> None: r = self.reader b = r.buffer - if ( - r.pos == 0 - and len(b) == 0 # this is something of a hack - and self.event[-1] == "\004" - ): - r.update_screen() - r.console.finish() - raise EOFError + if self.event[-1] == "\004": + if b and b[-1].endswith("\n"): + self.finish = True + elif ( + r.pos == 0 + and len(b) == 0 # this is something of a hack + ): + r.update_screen() + r.console.finish() + raise EOFError + + changed_from: int | None = None for i in range(r.get_arg()): if r.pos != len(b): del b[r.pos] - r.dirty = True + changed_from = r.pos if changed_from is None else min(changed_from, r.pos) else: self.reader.error("end of buffer") + if changed_from is not None: + r.invalidate_buffer(changed_from) class accept(FinishCommand): @@ -447,6 +464,7 @@ def do(self) -> None: with self.reader.suspend(): self.reader.msg = _sitebuiltins._Helper()() # type: ignore[assignment] + self.reader.invalidate_prompt() class invalid_key(Command): @@ -467,22 +485,24 @@ def do(self) -> None: from .pager import get_pager from site import gethistoryfile + # After the pager exits, the screen state is unknown (Unix may + # restore via alternate screen, Windows shows pager output). + # Clear and force a full redraw at the end for consistency. + self.reader.console.clear() + history = os.linesep.join(self.reader.history[:]) self.reader.console.restore() pager = get_pager() pager(history, gethistoryfile()) self.reader.console.prepare() - # We need to copy over the state so that it's consistent between - # console and reader, and console does not overwrite/append stuff - self.reader.console.screen = self.reader.screen.copy() - self.reader.console.posxy = self.reader.cxy + self.reader.invalidate_full() class paste_mode(Command): def do(self) -> None: self.reader.paste_mode = not self.reader.paste_mode - self.reader.dirty = True + self.reader.invalidate_prompt() class perform_bracketed_paste(Command): @@ -499,4 +519,3 @@ def do(self) -> None: s=time.time() - start, ) self.reader.insert(data.replace(done, "")) - self.reader.last_refresh_cache.invalidated = True diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py index 9d2d43be5144e8c..7eb3ebac84124b5 100644 --- a/Lib/_pyrepl/completing_reader.py +++ b/Lib/_pyrepl/completing_reader.py @@ -21,16 +21,18 @@ from __future__ import annotations from dataclasses import dataclass, field +from typing import TYPE_CHECKING import re from . import commands, console, reader +from .render import RenderLine, ScreenOverlay from .reader import Reader # types Command = commands.Command -if False: - from .types import KeySpec, CommandName +if TYPE_CHECKING: + from .types import CompletionAction, Keymap def prefix(wordlist: list[str], j: int = 0) -> str: @@ -168,39 +170,68 @@ def do(self) -> None: r: CompletingReader r = self.reader # type: ignore[assignment] last_is_completer = r.last_command_is(self.__class__) + if r.cmpltn_action: + if last_is_completer: # double-tab: execute action + msg = r.cmpltn_action[1]() + r.cmpltn_action = None # consumed + if msg: + r.msg = msg + r.cmpltn_message_visible = True + r.invalidate_message() + else: # other input since last tab: cancel action + r.cmpltn_action = None + immutable_completions = r.assume_immutable_completions completions_unchangable = last_is_completer and immutable_completions stem = r.get_stem() if not completions_unchangable: - r.cmpltn_menu_choices = r.get_completions(stem) + r.cmpltn_menu_choices, r.cmpltn_action = r.get_completions(stem) completions = r.cmpltn_menu_choices if not completions: - r.error("no matches") + if not r.cmpltn_action: + r.error("no matches") elif len(completions) == 1: - if completions_unchangable and len(completions[0]) == len(stem): + completion = stripcolor(completions[0]) + if completions_unchangable and len(completion) == len(stem): r.msg = "[ sole completion ]" - r.dirty = True - r.insert(completions[0][len(stem):]) + r.cmpltn_message_visible = True + r.invalidate_message() + r.insert(completion[len(stem):]) else: - p = prefix(completions, len(stem)) + clean_completions = [stripcolor(word) for word in completions] + p = prefix(clean_completions, len(stem)) if p: r.insert(p) if last_is_completer: r.cmpltn_menu_visible = True - r.cmpltn_message_visible = False r.cmpltn_menu, r.cmpltn_menu_end = build_menu( r.console, completions, r.cmpltn_menu_end, r.use_brackets, r.sort_in_column) - r.dirty = True + if r.msg: + r.msg = "" + r.cmpltn_message_visible = False + r.invalidate_message() + r.invalidate_overlay() elif not r.cmpltn_menu_visible: - r.cmpltn_message_visible = True - if stem + p in completions: + if stem + p in clean_completions: r.msg = "[ complete but not unique ]" - r.dirty = True + r.cmpltn_message_visible = True + r.invalidate_message() else: r.msg = "[ not unique ]" - r.dirty = True + r.cmpltn_message_visible = True + r.invalidate_message() + + if r.cmpltn_action: + if r.msg and r.cmpltn_message_visible: + # There is already a message (eg. [ not unique ]) that + # would conflict for next tab: cancel action + r.cmpltn_action = None + else: + r.msg = r.cmpltn_action[0] + r.cmpltn_message_visible = True + r.invalidate_message() class self_insert(commands.self_insert): @@ -215,11 +246,12 @@ def do(self) -> None: r.cmpltn_reset() else: completions = [w for w in r.cmpltn_menu_choices - if w.startswith(stem)] + if stripcolor(w).startswith(stem)] if completions: r.cmpltn_menu, r.cmpltn_menu_end = build_menu( r.console, completions, 0, r.use_brackets, r.sort_in_column) + r.invalidate_overlay() else: r.cmpltn_reset() @@ -240,6 +272,7 @@ class CompletingReader(Reader): cmpltn_message_visible: bool = field(init=False) cmpltn_menu_end: int = field(init=False) cmpltn_menu_choices: list[str] = field(init=False) + cmpltn_action: CompletionAction | None = field(init=False) def __post_init__(self) -> None: super().__post_init__() @@ -248,7 +281,7 @@ def __post_init__(self) -> None: self.commands[c.__name__] = c self.commands[c.__name__.replace('_', '-')] = c - def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + def collect_keymap(self) -> Keymap: return super().collect_keymap() + ( (r'\t', 'complete'),) @@ -257,30 +290,30 @@ def after_command(self, cmd: Command) -> None: if not isinstance(cmd, (complete, self_insert)): self.cmpltn_reset() - def calc_screen(self) -> list[str]: - screen = super().calc_screen() - if self.cmpltn_menu_visible: - # We display the completions menu below the current prompt - ly = self.lxy[1] + 1 - screen[ly:ly] = self.cmpltn_menu - # If we're not in the middle of multiline edit, don't append to screeninfo - # since that screws up the position calculation in pos2xy function. - # This is a hack to prevent the cursor jumping - # into the completions menu when pressing left or down arrow. - if self.pos != len(self.buffer): - self.screeninfo[ly:ly] = [(0, [])]*len(self.cmpltn_menu) - return screen + def get_screen_overlays(self) -> tuple[ScreenOverlay, ...]: + if not self.cmpltn_menu_visible: + return () + return ( + ScreenOverlay( + self.lxy[1] + 1, + tuple(RenderLine.from_rendered_text(line) for line in self.cmpltn_menu), + insert=True, + ), + ) def finish(self) -> None: super().finish() self.cmpltn_reset() def cmpltn_reset(self) -> None: + if getattr(self, "cmpltn_menu_visible", False): + self.invalidate_overlay() self.cmpltn_menu = [] self.cmpltn_menu_visible = False self.cmpltn_message_visible = False self.cmpltn_menu_end = 0 self.cmpltn_menu_choices = [] + self.cmpltn_action = None def get_stem(self) -> str: st = self.syntax_table @@ -291,8 +324,8 @@ def get_stem(self) -> str: p -= 1 return ''.join(b[p+1:self.pos]) - def get_completions(self, stem: str) -> list[str]: - return [] + def get_completions(self, stem: str) -> tuple[list[str], CompletionAction | None]: + return [], None def get_line(self) -> str: """Return the current line until the cursor position.""" diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index e0535d503963164..2a53d5ff581fa2c 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -19,23 +19,25 @@ from __future__ import annotations +import os import _colorize from abc import ABC, abstractmethod import ast import code import linecache -from dataclasses import dataclass, field -import os.path +from dataclasses import dataclass import re import sys +from typing import TYPE_CHECKING - -TYPE_CHECKING = False +from .render import RenderedScreen +from .trace import trace if TYPE_CHECKING: - from typing import IO - from typing import Callable + from typing import Callable, IO + + from .types import CursorXY @dataclass @@ -47,10 +49,17 @@ class Event: @dataclass class Console(ABC): - posxy: tuple[int, int] - screen: list[str] = field(default_factory=list) + posxy: CursorXY = (0, 0) height: int = 25 width: int = 80 + _redraw_debug_palette: tuple[str, ...] = ( + "\x1b[41m", + "\x1b[42m", + "\x1b[43m", + "\x1b[44m", + "\x1b[45m", + "\x1b[46m", + ) def __init__( self, @@ -71,8 +80,52 @@ def __init__( else: self.output_fd = f_out.fileno() + self.posxy = (0, 0) + self.height = 25 + self.width = 80 + self._rendered_screen = RenderedScreen.empty() + self._redraw_visual_cycle = 0 + + @property + def screen(self) -> list[str]: + return list(self._rendered_screen.screen_lines) + + def sync_rendered_screen( + self, + rendered_screen: RenderedScreen, + posxy: CursorXY | None = None, + ) -> None: + if posxy is None: + posxy = rendered_screen.cursor + self.posxy = posxy + self._rendered_screen = rendered_screen + trace( + "console.sync_rendered_screen lines={lines} cursor={cursor}", + lines=len(rendered_screen.composed_lines), + cursor=posxy, + ) + + def invalidate_render_state(self) -> None: + self._rendered_screen = RenderedScreen.empty() + trace("console.invalidate_render_state") + + def begin_redraw_visualization(self) -> str | None: + if "PYREPL_VISUALIZE_REDRAWS" not in os.environ: + return None + + palette = self._redraw_debug_palette + cycle = self._redraw_visual_cycle + style = palette[cycle % len(palette)] + self._redraw_visual_cycle = cycle + 1 + trace( + "console.begin_redraw_visualization cycle={cycle} style={style!r}", + cycle=cycle, + style=style, + ) + return style + @abstractmethod - def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: ... + def refresh(self, rendered_screen: RenderedScreen) -> None: ... @abstractmethod def prepare(self) -> None: ... diff --git a/Lib/_pyrepl/content.py b/Lib/_pyrepl/content.py new file mode 100644 index 000000000000000..3cb22fee84b7406 --- /dev/null +++ b/Lib/_pyrepl/content.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from .utils import ColorSpan, StyleRef, THEME, iter_display_chars, unbracket, wlen + + +@dataclass(frozen=True, slots=True) +class ContentFragment: + """A single display character with its visual width and style. + + The body of ``>>> def greet`` becomes one fragment per character:: + + d e f g r e e t + ╰──┴──╯ ╰──┴──┴──┴──╯ + keyword (unstyled) + + e.g. ``ContentFragment("d", 1, StyleRef(tag="keyword"))``. + """ + + text: str + width: int + style: StyleRef = StyleRef() + + +@dataclass(frozen=True, slots=True) +class PromptContent: + """The prompt split into leading full-width lines and an inline portion. + + For the common ``">>> "`` prompt (no newlines):: + + >>> def greet(name): + ╰─╯ + text=">>> ", width=4, leading_lines=() + + If ``sys.ps1`` contains newlines, e.g. ``"Python 3.13\\n>>> "``:: + + Python 3.13 ← leading_lines[0] + >>> def greet(name): + ╰─╯ + text=">>> ", width=4 + """ + + leading_lines: tuple[ContentFragment, ...] + text: str + width: int + + +@dataclass(frozen=True, slots=True) +class SourceLine: + """One logical line from the editor buffer, before styling. + + Given this two-line input in the REPL:: + + >>> def greet(name): + ... return name + ▲ cursor + + The buffer ``"def greet(name):\\n return name"`` yields:: + + SourceLine(lineno=0, text="def greet(name):", + start_offset=0, has_newline=True) + SourceLine(lineno=1, text=" return name", + start_offset=17, cursor_index=14) + """ + + lineno: int + text: str + start_offset: int + has_newline: bool + cursor_index: int | None = None + + @property + def cursor_on_line(self) -> bool: + return self.cursor_index is not None + + +@dataclass(frozen=True, slots=True) +class ContentLine: + """A logical line paired with its prompt and styled body. + + For ``>>> def greet(name):``:: + + >>> def greet(name): + ╰─╯ ╰──────────────╯ + prompt body: one ContentFragment per character + """ + + source: SourceLine + prompt: PromptContent + body: tuple[ContentFragment, ...] + + +def process_prompt(prompt: str) -> PromptContent: + r"""Return prompt content with width measured without zero-width markup.""" + + prompt_text = unbracket(prompt, including_content=False) + visible_prompt = unbracket(prompt, including_content=True) + leading_lines: list[ContentFragment] = [] + + while "\n" in prompt_text: + leading_text, _, prompt_text = prompt_text.partition("\n") + visible_leading, _, visible_prompt = visible_prompt.partition("\n") + leading_lines.append(ContentFragment(leading_text, wlen(visible_leading))) + + return PromptContent(tuple(leading_lines), prompt_text, wlen(visible_prompt)) + + +def build_body_fragments( + buffer: str, + colors: list[ColorSpan] | None, + start_index: int, +) -> tuple[ContentFragment, ...]: + """Convert a line's text into styled content fragments.""" + # Two separate loops to avoid the THEME() call in the common uncolored path. + if colors is None: + return tuple( + ContentFragment( + styled_char.text, + styled_char.width, + StyleRef(), + ) + for styled_char in iter_display_chars(buffer, colors, start_index) + ) + + theme = THEME() + return tuple( + ContentFragment( + styled_char.text, + styled_char.width, + StyleRef.from_tag(styled_char.tag, theme[styled_char.tag]) + if styled_char.tag + else StyleRef(), + ) + for styled_char in iter_display_chars(buffer, colors, start_index) + ) diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py index 0468b9a26702673..8d5bd183f213398 100644 --- a/Lib/_pyrepl/fancy_termios.py +++ b/Lib/_pyrepl/fancy_termios.py @@ -20,19 +20,25 @@ import termios +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import cast +else: + cast = lambda typ, val: val + + class TermState: - def __init__(self, tuples): - ( - self.iflag, - self.oflag, - self.cflag, - self.lflag, - self.ispeed, - self.ospeed, - self.cc, - ) = tuples + def __init__(self, attrs: list[int | list[bytes]]) -> None: + self.iflag = cast(int, attrs[0]) + self.oflag = cast(int, attrs[1]) + self.cflag = cast(int, attrs[2]) + self.lflag = cast(int, attrs[3]) + self.ispeed = cast(int, attrs[4]) + self.ospeed = cast(int, attrs[5]) + self.cc = cast(list[bytes], attrs[6]) - def as_list(self): + def as_list(self) -> list[int | list[bytes]]: return [ self.iflag, self.oflag, @@ -45,32 +51,32 @@ def as_list(self): self.cc[:], ] - def copy(self): + def copy(self) -> "TermState": return self.__class__(self.as_list()) -def tcgetattr(fd): +def tcgetattr(fd: int) -> TermState: return TermState(termios.tcgetattr(fd)) -def tcsetattr(fd, when, attrs): +def tcsetattr(fd: int, when: int, attrs: TermState) -> None: termios.tcsetattr(fd, when, attrs.as_list()) class Term(TermState): TS__init__ = TermState.__init__ - def __init__(self, fd=0): + def __init__(self, fd: int = 0) -> None: self.TS__init__(termios.tcgetattr(fd)) self.fd = fd - self.stack = [] + self.stack: list[list[int | list[bytes]]] = [] - def save(self): + def save(self) -> None: self.stack.append(self.as_list()) - def set(self, when=termios.TCSANOW): + def set(self, when: int = termios.TCSANOW) -> None: termios.tcsetattr(self.fd, when, self.as_list()) - def restore(self): + def restore(self) -> None: self.TS__init__(self.stack.pop()) self.set() diff --git a/Lib/_pyrepl/fancycompleter.py b/Lib/_pyrepl/fancycompleter.py new file mode 100644 index 000000000000000..ac4f0afdbc721c3 --- /dev/null +++ b/Lib/_pyrepl/fancycompleter.py @@ -0,0 +1,213 @@ +# Copyright 2010-2025 Antonio Cuni +# Daniel Hahler +# +# All Rights Reserved +"""Colorful tab completion for Python prompt""" +from __future__ import annotations + +from _colorize import ANSIColors, get_colors, get_theme +import rlcompleter +import keyword +import types + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any + from _colorize import Theme + + +def safe_getattr(obj, name): + # Mirror rlcompleter's safeguards so completion does not + # call properties or reify lazy module attributes. + if isinstance(getattr(type(obj), name, None), property): + return None + if (isinstance(obj, types.ModuleType) + and isinstance(obj.__dict__.get(name), types.LazyImportType) + ): + return obj.__dict__.get(name) + return getattr(obj, name, None) + + +def colorize_matches(names: list[str], values: list[Any], theme: Theme) -> list[str]: + return [ + _color_for_obj(name, obj, theme) + for name, obj in zip(names, values) + ] + +def _color_for_obj(name: str, value: Any, theme: Theme) -> str: + t = type(value) + color = _color_by_type(t, theme) + return f"{color}{name}{ANSIColors.RESET}" + + +def _color_by_type(t, theme): + typename = t.__name__ + # this is needed e.g. to turn method-wrapper into method_wrapper, + # because if we want _colorize.FancyCompleter to be "dataclassable" + # our keys need to be valid identifiers. + typename = typename.replace('-', '_').replace('.', '_') + return getattr(theme.fancycompleter, typename, ANSIColors.RESET) + + +class Completer(rlcompleter.Completer): + """ + When doing something like a.b., keep the full a.b.attr completion + stem so readline-style completion can keep refining the menu as you type. + + Optionally, display the various completions in different colors + depending on the type. + """ + def __init__( + self, + namespace=None, + *, + use_colors='auto', + consider_getitems=True, + ): + from _pyrepl import readline + rlcompleter.Completer.__init__(self, namespace) + if use_colors == 'auto': + # use colors only if we can + use_colors = get_colors().RED != "" + self.use_colors = use_colors + self.consider_getitems = consider_getitems + + if self.use_colors: + # In GNU readline, this prevents escaping of ANSI control + # characters in completion results. pyrepl's parse_and_bind() + # is a no-op, but pyrepl handles ANSI sequences natively + # via real_len()/stripcolor(). + readline.parse_and_bind('set dont-escape-ctrl-chars on') + self.theme = get_theme() + else: + self.theme = None + + if self.consider_getitems: + delims = readline.get_completer_delims() + delims = delims.replace('[', '') + delims = delims.replace(']', '') + readline.set_completer_delims(delims) + + def complete(self, text, state): + # if you press at the beginning of a line, insert an actual + # \t. Else, trigger completion. + if text == "": + return ('\t', None)[state] + else: + return rlcompleter.Completer.complete(self, text, state) + + def _callable_postfix(self, val, word): + # disable automatic insertion of '(' for global callables + return word + + def _callable_attr_postfix(self, val, word): + return rlcompleter.Completer._callable_postfix(self, val, word) + + def global_matches(self, text): + names = rlcompleter.Completer.global_matches(self, text) + prefix = commonprefix(names) + if prefix and prefix != text: + return [prefix] + + names.sort() + values = [] + for name in names: + clean_name = name.rstrip(': ') + if keyword.iskeyword(clean_name) or keyword.issoftkeyword(clean_name): + values.append(None) + else: + try: + values.append(eval(name, self.namespace)) + except Exception: + values.append(None) + if self.use_colors and names: + return self.colorize_matches(names, values) + return names + + def attr_matches(self, text): + try: + expr, attr, names, values = self._attr_matches(text) + except ValueError: + return [] + + if not names: + return [] + + if len(names) == 1: + # No coloring: when returning a single completion, readline + # inserts it directly into the prompt, so ANSI codes would + # appear as literal characters. + return [self._callable_attr_postfix(values[0], f'{expr}.{names[0]}')] + + prefix = commonprefix(names) + if prefix and prefix != attr: + return [f'{expr}.{prefix}'] # autocomplete prefix + + names = [f'{expr}.{name}' for name in names] + if self.use_colors: + return self.colorize_matches(names, values) + return names + + def _attr_matches(self, text): + expr, attr = text.rsplit('.', 1) + if '(' in expr or ')' in expr: # don't call functions + return expr, attr, [], [] + try: + thisobject = eval(expr, self.namespace) + except Exception: + return expr, attr, [], [] + + # get the content of the object, except __builtins__ + words = set(dir(thisobject)) - {'__builtins__'} + + if hasattr(thisobject, '__class__'): + words.add('__class__') + words.update(rlcompleter.get_class_members(thisobject.__class__)) + names = [] + values = [] + n = len(attr) + if attr == '': + noprefix = '_' + elif attr == '_': + noprefix = '__' + else: + noprefix = None + + # sort the words now to make sure to return completions in + # alphabetical order. It's easier to do it now, else we would need to + # sort 'names' later but make sure that 'values' in kept in sync, + # which is annoying. + words = sorted(words) + while True: + for word in words: + if ( + word[:n] == attr + and not (noprefix and word[:n+1] == noprefix) + ): + value = safe_getattr(thisobject, word) + names.append(word) + values.append(value) + if names or not noprefix: + break + if noprefix == '_': + noprefix = '__' + else: + noprefix = None + + return expr, attr, names, values + + def colorize_matches(self, names, values): + return colorize_matches(names, values, self.theme) + + +def commonprefix(names): + """Return the common prefix of all 'names'""" + if not names: + return '' + s1 = min(names) + s2 = max(names) + for i, c in enumerate(s1): + if c != s2[i]: + return s1[:i] + return s1 diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py index c4b95fa2e81ee60..09b969d80bc231f 100644 --- a/Lib/_pyrepl/historical_reader.py +++ b/Lib/_pyrepl/historical_reader.py @@ -90,7 +90,7 @@ def do(self) -> None: if r.get_unicode() != r.history[r.historyi]: r.buffer = list(r.history[r.historyi]) r.pos = len(r.buffer) - r.dirty = True + r.invalidate_buffer(0) class first_history(commands.Command): @@ -130,10 +130,11 @@ def do(self) -> None: o = len(r.yank_arg_yanked) else: o = 0 + start = r.pos - o b[r.pos - o : r.pos] = list(w) r.yank_arg_yanked = w r.pos += len(w) - o - r.dirty = True + r.invalidate_buffer(start) class forward_history_isearch(commands.Command): @@ -142,7 +143,7 @@ def do(self) -> None: r.isearch_direction = ISEARCH_DIRECTION_FORWARDS r.isearch_start = r.historyi, r.pos r.isearch_term = "" - r.dirty = True + r.invalidate_prompt() r.push_input_trans(r.isearch_trans) @@ -150,7 +151,7 @@ class reverse_history_isearch(commands.Command): def do(self) -> None: r = self.reader r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS - r.dirty = True + r.invalidate_prompt() r.isearch_term = "" r.push_input_trans(r.isearch_trans) r.isearch_start = r.historyi, r.pos @@ -163,7 +164,7 @@ def do(self) -> None: r.pop_input_trans() r.select_item(r.isearch_start[0]) r.pos = r.isearch_start[1] - r.dirty = True + r.invalidate_prompt() class isearch_add_character(commands.Command): @@ -171,7 +172,7 @@ def do(self) -> None: r = self.reader b = r.buffer r.isearch_term += self.event[-1] - r.dirty = True + r.invalidate_prompt() p = r.pos + len(r.isearch_term) - 1 if b[p : p + 1] != [r.isearch_term[-1]]: r.isearch_next() @@ -182,7 +183,7 @@ def do(self) -> None: r = self.reader if len(r.isearch_term) > 0: r.isearch_term = r.isearch_term[:-1] - r.dirty = True + r.invalidate_prompt() else: r.error("nothing to rubout") @@ -207,7 +208,7 @@ def do(self) -> None: r.isearch_direction = ISEARCH_DIRECTION_NONE r.console.forgetinput() r.pop_input_trans() - r.dirty = True + r.invalidate_prompt() @dataclass @@ -241,7 +242,6 @@ def __post_init__(self) -> None: isearch_end, isearch_add_character, isearch_cancel, - isearch_add_character, isearch_backspace, isearch_forwards, isearch_backwards, @@ -279,8 +279,7 @@ def select_item(self, i: int) -> None: self.buffer = list(buf) self.historyi = i self.pos = len(self.buffer) - self.dirty = True - self.last_refresh_cache.invalidated = True + self.invalidate_buffer(0) def get_item(self, i: int) -> str: if i != len(self.history): @@ -358,7 +357,7 @@ def search_next(self, *, forwards: bool) -> None: if forwards and not match_prefix: self.pos = 0 self.buffer = [] - self.dirty = True + self.invalidate_buffer(0) else: self.error("not found") return diff --git a/Lib/_pyrepl/input.py b/Lib/_pyrepl/input.py index 21c24eb5cde3e36..2d65246c700f274 100644 --- a/Lib/_pyrepl/input.py +++ b/Lib/_pyrepl/input.py @@ -38,10 +38,11 @@ from abc import ABC, abstractmethod import unicodedata from collections import deque +from typing import TYPE_CHECKING # types -if False: +if TYPE_CHECKING: from .types import EventTuple diff --git a/Lib/_pyrepl/layout.py b/Lib/_pyrepl/layout.py new file mode 100644 index 000000000000000..6d854d1142dd9f1 --- /dev/null +++ b/Lib/_pyrepl/layout.py @@ -0,0 +1,268 @@ +"""Wrap content lines to the terminal width before rendering.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Self + +from .content import ContentFragment, ContentLine +from .types import CursorXY, ScreenInfoRow + + +@dataclass(frozen=True, slots=True) +class LayoutRow: + """Metadata for one physical screen row. + + For the row ``>>> def greet(name):``:: + + >>> def greet(name): + ╰─╯ ╰──────────────╯ + 4 char_widths=(1,1,1,…) ← 16 entries + buffer_advance=17 ← includes the newline + """ + + prompt_width: int + char_widths: tuple[int, ...] + suffix_width: int = 0 + buffer_advance: int = 0 + + @property + def width(self) -> int: + return self.prompt_width + sum(self.char_widths) + self.suffix_width + + @property + def screeninfo(self) -> ScreenInfoRow: + widths = list(self.char_widths) + if self.suffix_width: + widths.append(self.suffix_width) + return self.prompt_width, widths + + +@dataclass(frozen=True, slots=True) +class LayoutMap: + """Mapping between buffer positions and screen coordinates. + + Single source of truth for cursor placement. Given:: + + >>> def greet(name): ← row 0, buffer_advance=17 + ... return name ← row 1, buffer_advance=15 + ▲cursor + + ``pos_to_xy(31)`` → ``(18, 1)``: prompt width 4 + 14 body chars. + """ + rows: tuple[LayoutRow, ...] + + @classmethod + def empty(cls) -> Self: + return cls((LayoutRow(0, ()),)) + + @property + def screeninfo(self) -> list[ScreenInfoRow]: + return [row.screeninfo for row in self.rows] + + def max_column(self, y: int) -> int: + return self.rows[y].width + + def max_row(self) -> int: + return len(self.rows) - 1 + + def pos_to_xy(self, pos: int) -> CursorXY: + if not self.rows: + return 0, 0 + + remaining = pos + for y, row in enumerate(self.rows): + if remaining <= len(row.char_widths): + # Prompt-only leading rows are terminal scenery, not real + # buffer positions. Treating them as real just manufactures + # bugs. + if remaining == 0 and not row.char_widths and row.buffer_advance == 0 and y < len(self.rows) - 1: + continue + x = row.prompt_width + for width in row.char_widths[:remaining]: + x += width + return x, y + remaining -= row.buffer_advance + last_row = self.rows[-1] + return last_row.width - last_row.suffix_width, len(self.rows) - 1 + + def xy_to_pos(self, x: int, y: int) -> int: + if not self.rows: + return 0 + + pos = 0 + for row in self.rows[:y]: + pos += row.buffer_advance + + row = self.rows[y] + cur_x = row.prompt_width + char_widths = row.char_widths + i = 0 + for i, width in enumerate(char_widths): + if cur_x >= x: + # Include trailing zero-width (combining) chars at this position + for trailing_width in char_widths[i:]: + if trailing_width == 0: + pos += 1 + else: + break + return pos + if width == 0: + pos += 1 + continue + cur_x += width + pos += 1 + return pos + + +@dataclass(frozen=True, slots=True) +class WrappedRow: + """One physical screen row after wrapping, ready for rendering. + + When a line overflows the terminal width, it splits into + multiple rows with a ``\\`` continuation marker:: + + >>> x = "a very long li\\ ← suffix="\\", suffix_width=1 + ne that wraps" ← prompt_text="" (continuation) + """ + prompt_text: str = "" + prompt_width: int = 0 + fragments: tuple[ContentFragment, ...] = () + layout_widths: tuple[int, ...] = () + suffix: str = "" + suffix_width: int = 0 + buffer_advance: int = 0 + + +@dataclass(frozen=True, slots=True) +class LayoutResult: + wrapped_rows: tuple[WrappedRow, ...] + layout_map: LayoutMap + line_end_offsets: tuple[int, ...] + + +def layout_content_lines( + lines: tuple[ContentLine, ...], + width: int, + start_offset: int, +) -> LayoutResult: + """Wrap content lines to fit *width* columns. + + A short line passes through as one ``WrappedRow``; a long line is + split at the column boundary with ``\\`` markers:: + + >>> short = 1 ← one WrappedRow + >>> x = "a long stri\\ ← two WrappedRows, first has suffix="\\" + ng" + """ + if width <= 0: + return LayoutResult((), LayoutMap(()), ()) + + offset = start_offset + wrapped_rows: list[WrappedRow] = [] + layout_rows: list[LayoutRow] = [] + line_end_offsets: list[int] = [] + + for line in lines: + newline_advance = int(line.source.has_newline) + for leading in line.prompt.leading_lines: + line_end_offsets.append(offset) + wrapped_rows.append( + WrappedRow( + fragments=(leading,), + ) + ) + layout_rows.append(LayoutRow(0, (), buffer_advance=0)) + + prompt_text = line.prompt.text + prompt_width = line.prompt.width + body = tuple(line.body) + body_widths = tuple(fragment.width for fragment in body) + + # Fast path: line fits on one row. + if not body_widths or (sum(body_widths) + prompt_width) < width: + offset += len(body) + newline_advance + line_end_offsets.append(offset) + wrapped_rows.append( + WrappedRow( + prompt_text=prompt_text, + prompt_width=prompt_width, + fragments=body, + layout_widths=body_widths, + buffer_advance=len(body) + newline_advance, + ) + ) + layout_rows.append( + LayoutRow( + prompt_width, + body_widths, + buffer_advance=len(body) + newline_advance, + ) + ) + continue + + # Slow path: line needs wrapping. + current_prompt = prompt_text + current_prompt_width = prompt_width + start = 0 + total = len(body) + while True: + # Find how many characters fit on this row. + index_to_wrap_before = 0 + column = 0 + for char_width in body_widths[start:]: + if column + char_width + current_prompt_width >= width: + break + index_to_wrap_before += 1 + column += char_width + + if index_to_wrap_before == 0 and start < total: + index_to_wrap_before = 1 # force progress + + at_line_end = (start + index_to_wrap_before) >= total + if at_line_end: + offset += index_to_wrap_before + newline_advance + suffix = "" + suffix_width = 0 + buffer_advance = index_to_wrap_before + newline_advance + else: + offset += index_to_wrap_before + suffix = "\\" + suffix_width = 1 + buffer_advance = index_to_wrap_before + + end = start + index_to_wrap_before + row_fragments = body[start:end] + row_widths = body_widths[start:end] + line_end_offsets.append(offset) + wrapped_rows.append( + WrappedRow( + prompt_text=current_prompt, + prompt_width=current_prompt_width, + fragments=row_fragments, + layout_widths=row_widths, + suffix=suffix, + suffix_width=suffix_width, + buffer_advance=buffer_advance, + ) + ) + layout_rows.append( + LayoutRow( + current_prompt_width, + row_widths, + suffix_width=suffix_width, + buffer_advance=buffer_advance, + ) + ) + + start = end + current_prompt = "" + current_prompt_width = 0 + if at_line_end: + break + + return LayoutResult( + tuple(wrapped_rows), + LayoutMap(tuple(layout_rows)), + tuple(line_end_offsets), + ) diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py index 1fddc63e3ee3ad0..92afaa5933a2250 100644 --- a/Lib/_pyrepl/pager.py +++ b/Lib/_pyrepl/pager.py @@ -138,7 +138,7 @@ def pipe_pager(text: str, cmd: str, title: str = '') -> None: '.' '?e (END):?pB %pB\\%..' ' (press h for help or q to quit)') - env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) + env['LESS'] = '-RcmPm{0}$PM{0}$'.format(prompt_string) proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, errors='backslashreplace', env=env) assert proc.stdin is not None diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 0ebd9162eca4bba..832e67b534f2969 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -25,16 +25,41 @@ import _colorize from contextlib import contextmanager -from dataclasses import dataclass, field, fields +from dataclasses import dataclass, field, fields, replace +from typing import Self from . import commands, console, input -from .utils import wlen, unbracket, disp_str, gen_colors, THEME +from .content import ( + ContentFragment, + ContentLine, + SourceLine, + build_body_fragments, + process_prompt as build_prompt_content, +) +from .layout import LayoutMap, LayoutResult, LayoutRow, WrappedRow, layout_content_lines +from .render import RenderCell, RenderLine, RenderedScreen, ScreenOverlay +from .utils import ANSI_ESCAPE_SEQUENCE, ColorSpan, THEME, StyleRef, gen_colors from .trace import trace # types Command = commands.Command -from .types import Callback, SimpleContextManager, KeySpec, CommandName +from collections.abc import Callable, Iterator +from .types import ( + Callback, + CommandName, + CursorXY, + Dimensions, + EventData, + KeySpec, + Keymap, + ScreenInfoRow, + SimpleContextManager, +) + +type CommandClass = type[Command] +type CommandInput = tuple[CommandName | CommandClass, EventData] +type PromptCellCacheKey = tuple[str, bool] # syntax classes @@ -52,8 +77,8 @@ def make_default_syntax_table() -> dict[str, int]: return st -def make_default_commands() -> dict[CommandName, type[Command]]: - result: dict[CommandName, type[Command]] = {} +def make_default_commands() -> dict[CommandName, CommandClass]: + result: dict[CommandName, CommandClass] = {} for v in vars(commands).values(): if isinstance(v, type) and issubclass(v, Command) and v.__name__[0].islower(): result[v.__name__] = v @@ -61,7 +86,7 @@ def make_default_commands() -> dict[CommandName, type[Command]]: return result -default_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( +default_keymap: Keymap = tuple( [ (r"\C-a", "beginning-of-line"), (r"\C-b", "left"), @@ -131,6 +156,77 @@ def make_default_commands() -> dict[CommandName, type[Command]]: ) +@dataclass(frozen=True, slots=True) +class RefreshInvalidation: + """Which parts of the screen need to be recomputed on the next refresh.""" + + cursor_only: bool = False + buffer_from_pos: int | None = None + prompt: bool = False + layout: bool = False + theme: bool = False + message: bool = False + overlay: bool = False + full: bool = False + + @classmethod + def empty(cls) -> Self: + return cls() + + @property + def needs_screen_refresh(self) -> bool: + return any( + ( + self.buffer_from_pos is not None, + self.prompt, + self.layout, + self.theme, + self.message, + self.overlay, + self.full, + ) + ) + + @property + def is_cursor_only(self) -> bool: + return self.cursor_only and not self.needs_screen_refresh + + @property + def buffer_rebuild_from_pos(self) -> int | None: + if self.full or self.prompt or self.layout or self.theme: + return 0 + return self.buffer_from_pos + + def with_cursor(self) -> Self: + if self.needs_screen_refresh: + return self + return replace(self, cursor_only=True) + + def with_buffer(self, from_pos: int) -> Self: + current = from_pos + if self.buffer_from_pos is not None: + current = min(current, self.buffer_from_pos) + return replace(self, cursor_only=False, buffer_from_pos=current) + + def with_prompt(self) -> Self: + return replace(self, cursor_only=False, prompt=True) + + def with_layout(self) -> Self: + return replace(self, cursor_only=False, layout=True) + + def with_theme(self) -> Self: + return replace(self, cursor_only=False, theme=True) + + def with_message(self) -> Self: + return replace(self, cursor_only=False, message=True) + + def with_overlay(self) -> Self: + return replace(self, cursor_only=False, overlay=True) + + def with_full(self) -> Self: + return replace(self, cursor_only=False, full=True) + + @dataclass(slots=True) class Reader: """The Reader class implements the bare bones of a command reader, @@ -148,10 +244,9 @@ class Reader: * pos: A 0-based index into 'buffer' for where the insertion point is. - * screeninfo: - A list of screen position tuples. Each list element is a tuple - representing information on visible line length for a given line. - Allows for efficient skipping of color escape sequences. + * layout: + A mapping between buffer positions and rendered rows/columns. + It is the internal source of truth for cursor placement. * cxy, lxy: the position of the insertion point in screen ... * syntax_table: @@ -162,8 +257,6 @@ class Reader: * arg: The emacs-style prefix argument. It will be None if no such argument has been provided. - * dirty: - True if we need to refresh the display. * kill_ring: The emacs-style kill-ring; manipulated with yank & yank-pop * ps1, ps2, ps3, ps4: @@ -198,66 +291,89 @@ class Reader: kill_ring: list[list[str]] = field(default_factory=list) msg: str = "" arg: int | None = None - dirty: bool = False finished: bool = False paste_mode: bool = False - commands: dict[str, type[Command]] = field(default_factory=make_default_commands) - last_command: type[Command] | None = None + commands: dict[CommandName, CommandClass] = field(default_factory=make_default_commands) + last_command: CommandClass | None = None syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table) - keymap: tuple[tuple[str, str], ...] = () + keymap: Keymap = () input_trans: input.KeymapTranslator = field(init=False) input_trans_stack: list[input.KeymapTranslator] = field(default_factory=list) - screen: list[str] = field(default_factory=list) - screeninfo: list[tuple[int, list[int]]] = field(init=False) - cxy: tuple[int, int] = field(init=False) - lxy: tuple[int, int] = field(init=False) - scheduled_commands: list[str] = field(default_factory=list) + rendered_screen: RenderedScreen = field(init=False) + layout: LayoutMap = field(init=False) + cxy: CursorXY = field(init=False) + lxy: CursorXY = field(init=False) + scheduled_commands: list[CommandName] = field(default_factory=list) can_colorize: bool = False + gen_colors: Callable[[str], Iterator[ColorSpan]] = gen_colors threading_hook: Callback | None = None + invalidation: RefreshInvalidation = field(init=False) ## cached metadata to speed up screen refreshes @dataclass class RefreshCache: - screen: list[str] = field(default_factory=list) - screeninfo: list[tuple[int, list[int]]] = field(init=False) + """Previously computed render/layout data for incremental refresh.""" + + render_lines: list[RenderLine] = field(default_factory=list) + layout_rows: list[LayoutRow] = field(default_factory=list) line_end_offsets: list[int] = field(default_factory=list) - pos: int = field(init=False) - cxy: tuple[int, int] = field(init=False) - dimensions: tuple[int, int] = field(init=False) - invalidated: bool = False + pos: int = 0 + dimensions: Dimensions = (0, 0) def update_cache(self, reader: Reader, - screen: list[str], - screeninfo: list[tuple[int, list[int]]], + render_lines: list[RenderLine], + layout_rows: list[LayoutRow], + line_end_offsets: list[int], ) -> None: - self.screen = screen.copy() - self.screeninfo = screeninfo.copy() + self.render_lines = render_lines.copy() + self.layout_rows = layout_rows.copy() + self.line_end_offsets = line_end_offsets.copy() self.pos = reader.pos - self.cxy = reader.cxy self.dimensions = reader.console.width, reader.console.height - self.invalidated = False def valid(self, reader: Reader) -> bool: - if self.invalidated: - return False dimensions = reader.console.width, reader.console.height dimensions_changed = dimensions != self.dimensions return not dimensions_changed - def get_cached_location(self, reader: Reader) -> tuple[int, int]: - if self.invalidated: - raise ValueError("Cache is invalidated") - offset = 0 - earliest_common_pos = min(reader.pos, self.pos) + def get_cached_location( + self, + reader: Reader, + buffer_from_pos: int | None = None, + *, + reuse_full: bool = False, + ) -> tuple[int, int]: + """Return (buffer_offset, num_reusable_lines) for incremental refresh. + + Three paths: + - reuse_full (overlay/message-only): reuse all cached lines. + - buffer_from_pos=None (full rebuild): rewind to common cursor pos. + - explicit buffer_from_pos: reuse lines before that position. + """ + if reuse_full: + if self.line_end_offsets: + last_offset = self.line_end_offsets[-1] + if last_offset >= len(reader.buffer): + return last_offset, len(self.line_end_offsets) + return 0, 0 + if buffer_from_pos is None: + buffer_from_pos = min(reader.pos, self.pos) num_common_lines = len(self.line_end_offsets) while num_common_lines > 0: - offset = self.line_end_offsets[num_common_lines - 1] - if earliest_common_pos > offset: + candidate = self.line_end_offsets[num_common_lines - 1] + if buffer_from_pos > candidate: break num_common_lines -= 1 - else: - offset = 0 + # Prompt-only leading rows consume no buffer content. Reusing them + # in isolation causes the next incremental rebuild to emit them a + # second time. + while ( + num_common_lines > 0 + and self.layout_rows[num_common_lines - 1].buffer_advance == 0 + ): + num_common_lines -= 1 + offset = self.line_end_offsets[num_common_lines - 1] if num_common_lines else 0 return offset, num_common_lines last_refresh_cache: RefreshCache = field(default_factory=RefreshCache) @@ -270,123 +386,267 @@ def __post_init__(self) -> None: self.input_trans = input.KeymapTranslator( self.keymap, invalid_cls="invalid-key", character_cls="self-insert" ) - self.screeninfo = [(0, [])] + self.layout = LayoutMap.empty() self.cxy = self.pos2xy() self.lxy = (self.pos, 0) + self.rendered_screen = RenderedScreen.empty() self.can_colorize = _colorize.can_colorize() + self.invalidation = RefreshInvalidation.empty() - self.last_refresh_cache.screeninfo = self.screeninfo + self.last_refresh_cache.layout_rows = list(self.layout.rows) self.last_refresh_cache.pos = self.pos - self.last_refresh_cache.cxy = self.cxy + self.last_refresh_cache.dimensions = (0, 0) - def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: - return default_keymap + @property + def screen(self) -> list[str]: + return list(self.rendered_screen.screen_lines) - def calc_screen(self) -> list[str]: - """Translate changes in self.buffer into changes in self.console.screen.""" - # Since the last call to calc_screen: - # screen and screeninfo may differ due to a completion menu being shown - # pos and cxy may differ due to edits, cursor movements, or completion menus + @property + def screeninfo(self) -> list[ScreenInfoRow]: + return self.layout.screeninfo - # Lines that are above both the old and new cursor position can't have changed, - # unless the terminal has been resized (which might cause reflowing) or we've - # entered or left paste mode (which changes prompts, causing reflowing). + def collect_keymap(self) -> Keymap: + return default_keymap + + def calc_screen(self) -> RenderedScreen: + """Translate the editable buffer into a base rendered screen.""" num_common_lines = 0 offset = 0 if self.last_refresh_cache.valid(self): - offset, num_common_lines = self.last_refresh_cache.get_cached_location(self) - - screen = self.last_refresh_cache.screen - del screen[num_common_lines:] - - screeninfo = self.last_refresh_cache.screeninfo - del screeninfo[num_common_lines:] - - last_refresh_line_end_offsets = self.last_refresh_cache.line_end_offsets - del last_refresh_line_end_offsets[num_common_lines:] + if ( + self.invalidation.buffer_from_pos is None + and not ( + self.invalidation.full + or self.invalidation.prompt + or self.invalidation.layout + or self.invalidation.theme + ) + and (self.invalidation.message or self.invalidation.overlay) + ): + # Fast path: only overlays or messages changed. + offset, num_common_lines = self.last_refresh_cache.get_cached_location( + self, + reuse_full=True, + ) + assert not self.last_refresh_cache.line_end_offsets or ( + self.last_refresh_cache.line_end_offsets[-1] >= len(self.buffer) + ), "Buffer modified without invalidate_buffer() call" + else: + offset, num_common_lines = self.last_refresh_cache.get_cached_location( + self, + self._buffer_refresh_from_pos(), + ) + + base_render_lines = self.last_refresh_cache.render_lines[:num_common_lines] + layout_rows = self.last_refresh_cache.layout_rows[:num_common_lines] + last_refresh_line_end_offsets = self.last_refresh_cache.line_end_offsets[:num_common_lines] + + source_lines = self._build_source_lines(offset, num_common_lines) + content_lines = self._build_content_lines( + source_lines, + prompt_from_cache=bool(offset and self.buffer[offset - 1] != "\n"), + ) + layout_result = self._layout_content(content_lines, offset) + base_render_lines.extend(self._render_wrapped_rows(layout_result.wrapped_rows)) + layout_rows.extend(layout_result.layout_map.rows) + last_refresh_line_end_offsets.extend(layout_result.line_end_offsets) - pos = self.pos - pos -= offset + self.layout = LayoutMap(tuple(layout_rows)) + self.cxy = self.pos2xy() + if not source_lines: + # reuse_full path: _build_source_lines didn't run, + # so lxy wasn't updated. Derive it from the buffer. + self.lxy = self._compute_lxy() + self.last_refresh_cache.update_cache( + self, + base_render_lines, + layout_rows, + last_refresh_line_end_offsets, + ) + return RenderedScreen(tuple(base_render_lines), self.cxy) - prompt_from_cache = (offset and self.buffer[offset - 1] != "\n") + def _buffer_refresh_from_pos(self) -> int: + """Return buffer position from which to rebuild content. - if self.can_colorize: - colors = list(gen_colors(self.get_unicode())) + Returns 0 (full rebuild) when no incremental position is known. + """ + buffer_from_pos = self.invalidation.buffer_rebuild_from_pos + if buffer_from_pos is not None: + return buffer_from_pos + return 0 + + def _compute_lxy(self) -> CursorXY: + """Derive logical cursor (col, lineno) from the buffer and pos.""" + text = "".join(self.buffer[:self.pos]) + lineno = text.count("\n") + if lineno: + col = self.pos - text.rindex("\n") - 1 else: - colors = None - trace("colors = {colors}", colors=colors) + col = self.pos + return col, lineno + + def _build_source_lines( + self, + offset: int, + first_lineno: int, + ) -> tuple[SourceLine, ...]: + if offset == len(self.buffer) and (offset > 0 or first_lineno > 0): + return () + + pos = self.pos - offset lines = "".join(self.buffer[offset:]).split("\n") cursor_found = False lines_beyond_cursor = 0 - for ln, line in enumerate(lines, num_common_lines): + source_lines: list[SourceLine] = [] + current_offset = offset + + for line_index, line in enumerate(lines): + lineno = first_lineno + line_index + has_newline = line_index < len(lines) - 1 line_len = len(line) + cursor_index: int | None = None if 0 <= pos <= line_len: - self.lxy = pos, ln + cursor_index = pos + self.lxy = pos, lineno cursor_found = True elif cursor_found: lines_beyond_cursor += 1 if lines_beyond_cursor > self.console.height: - # No need to keep formatting lines. - # The console can't show them. break + + source_lines.append( + SourceLine( + lineno=lineno, + text=line, + start_offset=current_offset, + has_newline=has_newline, + cursor_index=cursor_index, + ) + ) + pos -= line_len + 1 + current_offset += line_len + (1 if has_newline else 0) + + return tuple(source_lines) + + def _build_content_lines( + self, + source_lines: tuple[SourceLine, ...], + *, + prompt_from_cache: bool, + ) -> tuple[ContentLine, ...]: + if self.can_colorize: + colors = list(self.gen_colors(self.get_unicode())) + else: + colors = None + trace("colors = {colors}", colors=colors) + + content_lines: list[ContentLine] = [] + for source_line in source_lines: if prompt_from_cache: - # Only the first line's prompt can come from the cache prompt_from_cache = False prompt = "" else: - prompt = self.get_prompt(ln, line_len >= pos >= 0) - while "\n" in prompt: - pre_prompt, _, prompt = prompt.partition("\n") - last_refresh_line_end_offsets.append(offset) - screen.append(pre_prompt) - screeninfo.append((0, [])) - pos -= line_len + 1 - prompt, prompt_len = self.process_prompt(prompt) - chars, char_widths = disp_str(line, colors, offset) - wrapcount = (sum(char_widths) + prompt_len) // self.console.width - if wrapcount == 0 or not char_widths: - offset += line_len + 1 # Takes all of the line plus the newline - last_refresh_line_end_offsets.append(offset) - screen.append(prompt + "".join(chars)) - screeninfo.append((prompt_len, char_widths)) - else: - pre = prompt - prelen = prompt_len - for wrap in range(wrapcount + 1): - index_to_wrap_before = 0 - column = 0 - for char_width in char_widths: - if column + char_width + prelen >= self.console.width: - break - index_to_wrap_before += 1 - column += char_width - if len(chars) > index_to_wrap_before: - offset += index_to_wrap_before - post = "\\" - after = [1] - else: - offset += index_to_wrap_before + 1 # Takes the newline - post = "" - after = [] - last_refresh_line_end_offsets.append(offset) - render = pre + "".join(chars[:index_to_wrap_before]) + post - render_widths = char_widths[:index_to_wrap_before] + after - screen.append(render) - screeninfo.append((prelen, render_widths)) - chars = chars[index_to_wrap_before:] - char_widths = char_widths[index_to_wrap_before:] - pre = "" - prelen = 0 - self.screeninfo = screeninfo - self.cxy = self.pos2xy() - if self.msg: - for mline in self.msg.split("\n"): - screen.append(mline) - screeninfo.append((0, [])) - - self.last_refresh_cache.update_cache(self, screen, screeninfo) - return screen + prompt = self.get_prompt(source_line.lineno, source_line.cursor_on_line) + content_lines.append( + ContentLine( + source=source_line, + prompt=build_prompt_content(prompt), + body=build_body_fragments( + source_line.text, + colors, + source_line.start_offset, + ), + ) + ) + return tuple(content_lines) + + def _layout_content( + self, + content_lines: tuple[ContentLine, ...], + offset: int, + ) -> LayoutResult: + return layout_content_lines(content_lines, self.console.width, offset) + + def _render_wrapped_rows( + self, + wrapped_rows: tuple[WrappedRow, ...], + ) -> list[RenderLine]: + return [ + self._render_line( + row.prompt_text, + row.fragments, + row.suffix, + ) + for row in wrapped_rows + ] + + def _render_message_lines(self) -> tuple[RenderLine, ...]: + if not self.msg: + return () + width = self.console.width + render_lines: list[RenderLine] = [] + for message_line in self.msg.split("\n"): + # If self.msg is larger than console width, make it fit. + # TODO: try to split between words? + if not message_line: + render_lines.append(RenderLine.from_rendered_text("")) + continue + for offset in range(0, len(message_line), width): + render_lines.append( + RenderLine.from_rendered_text(message_line[offset : offset + width]) + ) + return tuple(render_lines) + + def get_screen_overlays(self) -> tuple[ScreenOverlay, ...]: + return () + + def compose_rendered_screen(self, base_screen: RenderedScreen) -> RenderedScreen: + overlays = list(self.get_screen_overlays()) + message_lines = self._render_message_lines() + if message_lines: + overlays.append(ScreenOverlay(len(base_screen.lines), message_lines)) + if not overlays: + return base_screen + return RenderedScreen(base_screen.lines, base_screen.cursor, tuple(overlays)) + + _prompt_cell_cache: dict[PromptCellCacheKey, tuple[RenderCell, ...]] = field( + init=False, default_factory=dict, repr=False + ) + + def _render_line( + self, + prefix: str, + fragments: tuple[ContentFragment, ...], + suffix: str = "", + ) -> RenderLine: + cells: list[RenderCell] = [] + if prefix: + cache_key = (prefix, self.can_colorize) + cached = self._prompt_cell_cache.get(cache_key) + if cached is None: + prompt_cells = RenderLine.from_rendered_text(prefix).cells + if self.can_colorize and prompt_cells and not ANSI_ESCAPE_SEQUENCE.search(prefix): + prompt_style = StyleRef.from_tag("prompt", THEME()["prompt"]) + prompt_cells = tuple( + RenderCell( + cell.text, + cell.width, + style=prompt_style if cell.text else cell.style, + controls=cell.controls, + ) + for cell in prompt_cells + ) + self._prompt_cell_cache[cache_key] = prompt_cells + cached = prompt_cells + cells.extend(cached) + cells.extend( + RenderCell(fragment.text, fragment.width, style=fragment.style) + for fragment in fragments + ) + if suffix: + cells.extend(RenderLine.from_rendered_text(suffix).cells) + return RenderLine.from_cells(cells) @staticmethod def process_prompt(prompt: str) -> tuple[str, int]: @@ -396,9 +656,8 @@ def process_prompt(prompt: str) -> tuple[str, int]: (\x01 and \x02) removed. The length ignores anything between those brackets as well as any ANSI escape sequences. """ - out_prompt = unbracket(prompt, including_content=False) - visible_prompt = unbracket(prompt, including_content=True) - return out_prompt, wlen(visible_prompt) + prompt_content = build_prompt_content(prompt) + return prompt_content.text, prompt_content.width def bow(self, p: int | None = None) -> int: """Return the 0-based index of the word break preceding p most @@ -460,10 +719,10 @@ def eol(self, p: int | None = None) -> int: def max_column(self, y: int) -> int: """Return the last x-offset for line y""" - return self.screeninfo[y][0] + sum(self.screeninfo[y][1]) + return self.layout.max_column(y) def max_row(self) -> int: - return len(self.screeninfo) - 1 + return self.layout.max_row() def get_arg(self, default: int = 1) -> int: """Return any prefix argument that the user has supplied, @@ -489,10 +748,6 @@ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: prompt = self.ps3 else: prompt = self.ps1 - - if self.can_colorize: - t = THEME() - prompt = f"{t.prompt}{prompt}{t.reset}" return prompt def push_input_trans(self, itrans: input.KeymapTranslator) -> None: @@ -504,65 +759,48 @@ def pop_input_trans(self) -> None: def setpos_from_xy(self, x: int, y: int) -> None: """Set pos according to coordinates x, y""" - pos = 0 - i = 0 - while i < y: - prompt_len, char_widths = self.screeninfo[i] - offset = len(char_widths) - in_wrapped_line = prompt_len + sum(char_widths) >= self.console.width - if in_wrapped_line: - pos += offset - 1 # -1 cause backslash is not in buffer - else: - pos += offset + 1 # +1 cause newline is in buffer - i += 1 - - j = 0 - cur_x = self.screeninfo[i][0] - while cur_x < x: - if self.screeninfo[i][1][j] == 0: - j += 1 # prevent potential future infinite loop - continue - cur_x += self.screeninfo[i][1][j] - j += 1 - pos += 1 - - self.pos = pos + self.pos = self.layout.xy_to_pos(x, y) - def pos2xy(self) -> tuple[int, int]: + def pos2xy(self) -> CursorXY: """Return the x, y coordinates of position 'pos'.""" + assert 0 <= self.pos <= len(self.buffer) + return self.layout.pos_to_xy(self.pos) + + def insert(self, text: str | list[str]) -> None: + """Insert 'text' at the insertion point.""" + start = self.pos + self.buffer[self.pos : self.pos] = list(text) + self.pos += len(text) + self.invalidate_buffer(start) - prompt_len, y = 0, 0 - char_widths: list[int] = [] - pos = self.pos - assert 0 <= pos <= len(self.buffer) + def invalidate_cursor(self) -> None: + self.invalidation = self.invalidation.with_cursor() - # optimize for the common case: typing at the end of the buffer - if pos == len(self.buffer) and len(self.screeninfo) > 0: - y = len(self.screeninfo) - 1 - prompt_len, char_widths = self.screeninfo[y] - return prompt_len + sum(char_widths), y + def invalidate_buffer(self, from_pos: int) -> None: + self.invalidation = self.invalidation.with_buffer(from_pos) - for prompt_len, char_widths in self.screeninfo: - offset = len(char_widths) - in_wrapped_line = prompt_len + sum(char_widths) >= self.console.width - if in_wrapped_line: - offset -= 1 # need to remove line-wrapping backslash + def invalidate_prompt(self) -> None: + self._prompt_cell_cache.clear() + self.invalidation = self.invalidation.with_prompt() - if offset >= pos: - break + def invalidate_layout(self) -> None: + self.invalidation = self.invalidation.with_layout() - if not in_wrapped_line: - offset += 1 # there's a newline in buffer + def invalidate_theme(self) -> None: + self._prompt_cell_cache.clear() + self.invalidation = self.invalidation.with_theme() - pos -= offset - y += 1 - return prompt_len + sum(char_widths[:pos]), y + def invalidate_message(self) -> None: + self.invalidation = self.invalidation.with_message() - def insert(self, text: str | list[str]) -> None: - """Insert 'text' at the insertion point.""" - self.buffer[self.pos : self.pos] = list(text) - self.pos += len(text) - self.dirty = True + def invalidate_overlay(self) -> None: + self.invalidation = self.invalidation.with_overlay() + + def invalidate_full(self) -> None: + self.invalidation = self.invalidation.with_full() + + def clear_invalidation(self) -> None: + self.invalidation = RefreshInvalidation.empty() def update_cursor(self) -> None: """Move the cursor to reflect changes in self.pos""" @@ -574,7 +812,7 @@ def after_command(self, cmd: Command) -> None: """This function is called to allow post command cleanup.""" if getattr(cmd, "kills_digit_arg", True): if self.arg is not None: - self.dirty = True + self.invalidate_prompt() self.arg = None def prepare(self) -> None: @@ -587,9 +825,15 @@ def prepare(self) -> None: self.finished = False del self.buffer[:] self.pos = 0 - self.dirty = True + self.layout = LayoutMap.empty() + self.cxy = self.pos2xy() + self.lxy = (self.pos, 0) + self.rendered_screen = RenderedScreen.empty() + self.invalidate_full() self.last_command = None - self.calc_screen() + base_screen = self.calc_screen() + self.rendered_screen = self.compose_rendered_screen(base_screen) + self.invalidation = RefreshInvalidation.empty() except BaseException: self.restore() raise @@ -598,7 +842,7 @@ def prepare(self) -> None: cmd = self.scheduled_commands.pop() self.do_cmd((cmd, [])) - def last_command_is(self, cls: type) -> bool: + def last_command_is(self, cls: CommandClass) -> bool: if not self.last_command: return False return issubclass(cls, self.last_command) @@ -619,27 +863,51 @@ def suspend(self) -> SimpleContextManager: setattr(self, arg, prev_state[arg]) self.prepare() + @contextmanager + def suspend_colorization(self) -> SimpleContextManager: + try: + old_can_colorize = self.can_colorize + self.can_colorize = False + yield + finally: + self.can_colorize = old_can_colorize + def finish(self) -> None: """Called when a command signals that we're finished.""" pass def error(self, msg: str = "none") -> None: self.msg = "! " + msg + " " - self.dirty = True + self.invalidate_message() self.console.beep() def update_screen(self) -> None: - if self.dirty: + if self.invalidation.is_cursor_only: + self.update_cursor() + self.clear_invalidation() + elif self.invalidation.needs_screen_refresh: self.refresh() def refresh(self) -> None: """Recalculate and refresh the screen.""" + self.console.height, self.console.width = self.console.getheightwidth() # this call sets up self.cxy, so call it first. - self.screen = self.calc_screen() - self.console.refresh(self.screen, self.cxy) - self.dirty = False + base_screen = self.calc_screen() + rendered_screen = self.compose_rendered_screen(base_screen) + self.rendered_screen = rendered_screen + trace( + "reader.refresh cursor={cursor} lines={lines} " + "dims=({width},{height}) invalidation={invalidation}", + cursor=self.cxy, + lines=len(rendered_screen.composed_lines), + width=self.console.width, + height=self.console.height, + invalidation=self.invalidation, + ) + self.console.refresh(rendered_screen) + self.clear_invalidation() - def do_cmd(self, cmd: tuple[str, list[str]]) -> None: + def do_cmd(self, cmd: CommandInput) -> None: """`cmd` is a tuple of "event_name" and "event", which in the current implementation is always just the "buffer" which happens to be a list of single-character strings.""" @@ -656,13 +924,14 @@ def do_cmd(self, cmd: tuple[str, list[str]]) -> None: command.do() self.after_command(command) - - if self.dirty: - self.refresh() - else: - self.update_cursor() - - if not isinstance(cmd, commands.digit_arg): + if ( + not self.invalidation.needs_screen_refresh + and not self.invalidation.is_cursor_only + ): + self.invalidate_cursor() + self.update_screen() + + if command_type is not commands.digit_arg: self.last_command = command_type self.finished = bool(command.finish) @@ -695,7 +964,7 @@ def handle1(self, block: bool = True) -> bool: if self.msg: self.msg = "" - self.dirty = True + self.invalidate_message() while True: # We use the same timeout as in readline.c: 100ms @@ -712,9 +981,13 @@ def handle1(self, block: bool = True) -> bool: if event.evt == "key": self.input_trans.push(event) elif event.evt == "scroll": + self.invalidate_full() self.refresh() + return True elif event.evt == "resize": + self.invalidate_full() self.refresh() + return True else: translate = False diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 23b8fa6b9c7625e..e4370b0d1462eaf 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -37,9 +37,10 @@ from rlcompleter import Completer as RLCompleter from . import commands, historical_reader -from .completing_reader import CompletingReader +from .completing_reader import CompletingReader, stripcolor from .console import Console as ConsoleType from ._module_completer import ModuleCompleter, make_default_module_completer +from .fancycompleter import Completer as FancyCompleter, colorize_matches Console: type[ConsoleType] _error: tuple[type[Exception], ...] | type[Exception] @@ -55,7 +56,7 @@ # types Command = commands.Command from collections.abc import Callable, Collection -from .types import Callback, Completer, KeySpec, CommandName +from .types import Callback, Completer, KeySpec, CommandName, CompletionAction TYPE_CHECKING = False @@ -103,6 +104,7 @@ class ReadlineConfig: readline_completer: Completer | None = None completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") module_completer: ModuleCompleter = field(default_factory=make_default_module_completer) + colorize_completions: Callable[[list[str], list[Any]], list[str]] | None = None @dataclass(kw_only=True) class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader): @@ -134,7 +136,7 @@ def get_stem(self) -> str: p -= 1 return "".join(b[p + 1 : self.pos]) - def get_completions(self, stem: str) -> list[str]: + def get_completions(self, stem: str) -> tuple[list[str], CompletionAction | None]: module_completions = self.get_module_completions() if module_completions is not None: return module_completions @@ -144,7 +146,7 @@ def get_completions(self, stem: str) -> list[str]: while p > 0 and b[p - 1] != "\n": p -= 1 num_spaces = 4 - ((self.pos - p) % 4) - return [" " * num_spaces] + return [" " * num_spaces], None result = [] function = self.config.readline_completer if function is not None: @@ -162,14 +164,23 @@ def get_completions(self, stem: str) -> list[str]: break result.append(next) state += 1 - # emulate the behavior of the standard readline that sorts - # the completions before displaying them. - result.sort() - return result - - def get_module_completions(self) -> list[str] | None: - line = self.get_line() - return self.config.module_completer.get_completions(line) + # Emulate readline's sorting using the visible text rather than + # the raw ANSI escape sequences used for colorized matches. + result.sort(key=stripcolor) + return result, None + + def get_module_completions(self) -> tuple[list[str], CompletionAction | None] | None: + line = stripcolor(self.get_line()) + colorize_completions = self.config.colorize_completions + result = self.config.module_completer.get_completions( + line, include_values=bool(colorize_completions) + ) + if result is None: + return None + names, values, action = result + if colorize_completions: + names = colorize_completions(names, values) + return names, action def get_trimmed_history(self, maxlength: int) -> list[str]: if maxlength >= 0: @@ -276,7 +287,7 @@ class maybe_accept(commands.Command): def do(self) -> None: r: ReadlineAlikeReader r = self.reader # type: ignore[assignment] - r.dirty = True # this is needed to hide the completion menu, if visible + r.invalidate_overlay() # hide completion menu, if visible # if there are already several lines and the cursor # is not on the last one, always insert a new \n. @@ -336,7 +347,7 @@ def do(self) -> None: break r.pos -= repeat del b[r.pos : r.pos + repeat] - r.dirty = True + r.invalidate_buffer(r.pos) else: self.reader.error("can't backspace at start") @@ -412,8 +423,12 @@ def set_completer_delims(self, delimiters: Collection[str]) -> None: def get_completer_delims(self) -> str: return "".join(sorted(self.config.completer_delims)) - def _histline(self, line: str) -> str: + def _histline(self, line: str, *, sanitize_nuls: bool = False) -> str: line = line.rstrip("\n") + if "\0" in line: + if not sanitize_nuls: + raise ValueError("embedded null character") + line = line.replace("\0", "") return line def get_history_length(self) -> int: @@ -446,9 +461,12 @@ def read_history_file(self, filename: str = gethistoryfile()) -> None: if line.endswith("\r"): buffer.append(line+'\n') else: - line = self._histline(line) + line = self._histline(line, sanitize_nuls=True) if buffer: - line = self._histline("".join(buffer).replace("\r", "") + line) + line = self._histline( + "".join(buffer).replace("\r", "") + line, + sanitize_nuls=True, + ) del buffer[:] if line: history.append(line) @@ -608,8 +626,19 @@ def _setup(namespace: Mapping[str, Any]) -> None: # set up namespace in rlcompleter, which requires it to be a bona fide dict if not isinstance(namespace, dict): namespace = dict(namespace) + use_basic_completer = ( + not sys.flags.ignore_environment + and os.getenv("PYTHON_BASIC_COMPLETER") + ) + completer_cls = RLCompleter if use_basic_completer else FancyCompleter + completer = completer_cls(namespace) + _wrapper.config.readline_completer = completer.complete + if isinstance(completer, FancyCompleter) and completer.use_colors: + theme = completer.theme + def _colorize(names: list[str], values: list[object]) -> list[str]: + return colorize_matches(names, values, theme) + _wrapper.config.colorize_completions = _colorize _wrapper.config.module_completer = ModuleCompleter(namespace) - _wrapper.config.readline_completer = RLCompleter(namespace).complete # this is not really what readline.c does. Better than nothing I guess import builtins diff --git a/Lib/_pyrepl/render.py b/Lib/_pyrepl/render.py new file mode 100644 index 000000000000000..b821f35d850825e --- /dev/null +++ b/Lib/_pyrepl/render.py @@ -0,0 +1,397 @@ +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field +from typing import Literal, Protocol, Self + +from .utils import ANSI_ESCAPE_SEQUENCE, THEME, StyleRef, str_width +from .types import CursorXY + +type RenderStyle = StyleRef | str | None +type LineUpdateKind = Literal[ + "insert_char", + "replace_char", + "replace_span", + "delete_then_insert", + "rewrite_suffix", +] + + +class _ThemeSyntax(Protocol): + """Protocol for theme objects that map tag names to SGR escape strings.""" + def __getitem__(self, key: str, /) -> str: ... + + +@dataclass(frozen=True, slots=True) +class RenderCell: + """One terminal cell: a character, its column width, and SGR style. + + A screen row like ``>>> def`` is a sequence of cells:: + + > > > d e f + ╰─╯╰─╯╰─╯╰─╯╰─╯╰─╯╰─╯ + """ + + text: str + width: int + style: StyleRef = field(default_factory=StyleRef) + controls: tuple[str, ...] = () + + @property + def terminal_text(self) -> str: + return render_cells((self,)) + + +def _theme_style(theme: _ThemeSyntax, tag: str) -> str: + return theme[tag] + + +def _style_escape(style: StyleRef) -> str: + if style.sgr: + return style.sgr + if style.tag is None: + return "" + return _theme_style(THEME(), style.tag) + + +def _update_terminal_state(state: str, escape: str) -> str: + if escape in {"\x1b[0m", "\x1b[m"}: + return "" + return state + escape + + +def _cells_from_rendered_text(text: str) -> tuple[RenderCell, ...]: + if not text: + return () + + cells: list[RenderCell] = [] + pending_controls: list[str] = [] + active_sgr = "" + index = 0 + + def append_plain_text(segment: str) -> None: + nonlocal pending_controls + if not segment: + return + if pending_controls: + cells.append(RenderCell("", 0, controls=tuple(pending_controls))) + pending_controls = [] + for char in segment: + cells.append( + RenderCell( + char, + str_width(char), + style=StyleRef.from_sgr(active_sgr), + ) + ) + + for match in ANSI_ESCAPE_SEQUENCE.finditer(text): + append_plain_text(text[index : match.start()]) + escape = match.group(0) + if escape.endswith("m"): + active_sgr = _update_terminal_state(active_sgr, escape) + else: + pending_controls.append(escape) + index = match.end() + + append_plain_text(text[index:]) + if pending_controls: + cells.append(RenderCell("", 0, controls=tuple(pending_controls))) + + return tuple(cells) + + +@dataclass(frozen=True, slots=True) +class RenderLine: + """One physical screen row as a tuple of :class:`RenderCell` objects. + + ``text`` is the pre-rendered terminal string (characters + SGR escapes); + ``width`` is the total visible column count. + """ + + cells: tuple[RenderCell, ...] + text: str + width: int + + @classmethod + def from_cells(cls, cells: Iterable[RenderCell]) -> Self: + cell_tuple = tuple(cells) + return cls( + cells=cell_tuple, + text=render_cells(cell_tuple), + width=sum(cell.width for cell in cell_tuple), + ) + + @classmethod + def from_parts( + cls, + parts: Sequence[str], + widths: Sequence[int], + styles: Sequence[RenderStyle] | None = None, + ) -> Self: + if styles is None: + return cls.from_cells( + RenderCell(text, width) + for text, width in zip(parts, widths) + ) + + cells: list[RenderCell] = [] + for text, width, style in zip(parts, widths, styles): + if isinstance(style, StyleRef): + cells.append(RenderCell(text, width, style=style)) + elif style is None: + cells.append(RenderCell(text, width)) + else: + cells.append(RenderCell(text, width, style=StyleRef.from_tag(style))) + return cls.from_cells(cells) + + @classmethod + def from_rendered_text(cls, text: str) -> Self: + return cls.from_cells(_cells_from_rendered_text(text)) + + +@dataclass(frozen=True, slots=True) +class ScreenOverlay: + """An overlay that replaces or inserts lines at a screen position. + + If *insert* is True, lines are spliced in (shifting content down); + if False (default), lines replace existing content at *y*. + + Overlays are used to display tab completion menus and status messages. + For example, a tab-completion menu inserted below the input:: + + >>> os.path.j ← line 0 (base content) + join ← ScreenOverlay(y=1, insert=True) + junction ← (pushes remaining lines down) + ... ← line 1 (shifted down by 2) + """ + y: int + lines: tuple[RenderLine, ...] + insert: bool = False + + +@dataclass(frozen=True, slots=True) +class RenderedScreen: + """The complete screen state: content lines, cursor, and overlays. + + ``lines`` holds the base content; ``composed_lines`` is the final + result after overlays (completion menus, messages) are applied:: + + lines: composed_lines: + ┌──────────────────┐ ┌──────────────────┐ + │>>> os.path.j │ │>>> os.path.j │ + │... │ ──► │ join │ ← overlay + └──────────────────┘ │... │ + └──────────────────┘ + """ + + lines: tuple[RenderLine, ...] + cursor: CursorXY + overlays: tuple[ScreenOverlay, ...] = () + composed_lines: tuple[RenderLine, ...] = field(init=False, default=()) + + def __post_init__(self) -> None: + object.__setattr__(self, "composed_lines", self._compose()) + + def _compose(self) -> tuple[RenderLine, ...]: + """Apply overlays in tuple order; inserts shift subsequent positions.""" + if not self.overlays: + return self.lines + + lines = list(self.lines) + y_offset = 0 + for overlay in self.overlays: + adjusted_y = overlay.y + y_offset + assert adjusted_y >= 0, ( + f"Overlay y={overlay.y} with offset={y_offset} is negative; " + "overlays must be sorted by ascending y" + ) + if overlay.insert: + # Splice overlay lines in, pushing existing content down. + lines[adjusted_y:adjusted_y] = overlay.lines + y_offset += len(overlay.lines) + else: + # Replace existing lines at the overlay position. + target_len = adjusted_y + len(overlay.lines) + if len(lines) < target_len: + lines.extend([EMPTY_RENDER_LINE] * (target_len - len(lines))) + for index, line in enumerate(overlay.lines): + lines[adjusted_y + index] = line + return tuple(lines) + + @classmethod + def empty(cls) -> Self: + return cls((), (0, 0), ()) + + @classmethod + def from_screen_lines( + cls, + screen: Sequence[str], + cursor: CursorXY, + ) -> Self: + return cls( + tuple(RenderLine.from_rendered_text(line) for line in screen), + cursor, + (), + ) + + def with_overlay( + self, + y: int, + lines: Iterable[RenderLine], + ) -> Self: + return type(self)( + self.lines, + self.cursor, + self.overlays + (ScreenOverlay(y, tuple(lines)),), + ) + + @property + def screen_lines(self) -> tuple[str, ...]: + return tuple(line.text for line in self.composed_lines) + + +@dataclass(frozen=True, slots=True) +class LineDiff: + """The changed region between an old and new version of one screen row. + + When the user types ``e`` so the row changes from + ``>>> nam`` to ``>>> name``:: + + >>> n a m old + >>> n a m e new + ╰─╯ + start_cell=7, new_cells=("m","e"), old_cells=("m",) + """ + + start_cell: int + start_x: int + old_cells: tuple[RenderCell, ...] + new_cells: tuple[RenderCell, ...] + old_width: int + new_width: int + + @property + def old_text(self) -> str: + return render_cells(self.old_cells) + + @property + def new_text(self) -> str: + return render_cells(self.new_cells) + + @property + def old_changed_width(self) -> int: + return sum(cell.width for cell in self.old_cells) + + @property + def new_changed_width(self) -> int: + return sum(cell.width for cell in self.new_cells) + + +EMPTY_RENDER_LINE = RenderLine(cells=(), text="", width=0) + + +@dataclass(frozen=True, slots=True) +class LineUpdate: + kind: LineUpdateKind + y: int + start_cell: int + start_x: int + """Screen x-coordinate where the update begins. Used for cursor positioning.""" + cells: tuple[RenderCell, ...] + char_width: int = 0 + clear_eol: bool = False + reset_to_margin: bool = False + """If True, the console must resync the cursor position after writing + (needed when cells contain non-SGR escape sequences that may move the cursor).""" + text: str = field(init=False, default="") + + def __post_init__(self) -> None: + object.__setattr__(self, "text", render_cells(self.cells)) + + +def _controls_require_cursor_resync(controls: Sequence[str]) -> bool: + # Anything beyond SGR means the cursor may no longer be where we left it. + return any(not control.endswith("m") for control in controls) + + +def requires_cursor_resync(cells: Sequence[RenderCell]) -> bool: + return any(_controls_require_cursor_resync(cell.controls) for cell in cells) + + +def render_cells( + cells: Sequence[RenderCell], + visual_style: str | None = None, +) -> str: + """Render a sequence of cells into a terminal string with SGR escapes. + + Tracks the active SGR state to emit resets only when the style + actually changes, minimizing output bytes. + + If *visual_style* is given (used by redraw visualization), it is appended + to every cell's style. + """ + rendered: list[str] = [] + active_escape = "" + for cell in cells: + if cell.controls: + rendered.extend(cell.controls) + if not cell.text: + continue + + target_escape = _style_escape(cell.style) + if visual_style is not None: + target_escape += visual_style + if target_escape != active_escape: + if active_escape: + rendered.append("\x1b[0m") + if target_escape: + rendered.append(target_escape) + active_escape = target_escape + rendered.append(cell.text) + + if active_escape: + rendered.append("\x1b[0m") + return "".join(rendered) + + +def diff_render_lines(old: RenderLine, new: RenderLine) -> LineDiff | None: + if old == new: + return None + + prefix = 0 + start_x = 0 + max_prefix = min(len(old.cells), len(new.cells)) + while prefix < max_prefix and old.cells[prefix] == new.cells[prefix]: + # Stop at any cell with non-SGR controls, since those might affect + # cursor position and must be re-emitted. + if old.cells[prefix].controls: + break + start_x += old.cells[prefix].width + prefix += 1 + + old_suffix = len(old.cells) + new_suffix = len(new.cells) + while old_suffix > prefix and new_suffix > prefix: + old_cell = old.cells[old_suffix - 1] + new_cell = new.cells[new_suffix - 1] + if old_cell.controls or new_cell.controls or old_cell != new_cell: + break + old_suffix -= 1 + new_suffix -= 1 + + # Extend diff range to include trailing zero-width combining characters, + # so we never render a combining char without its base character. + while old_suffix < len(old.cells) and old.cells[old_suffix].width == 0: + old_suffix += 1 + while new_suffix < len(new.cells) and new.cells[new_suffix].width == 0: + new_suffix += 1 + + return LineDiff( + start_cell=prefix, + start_x=start_x, + old_cells=old.cells[prefix:old_suffix], + new_cells=new.cells[prefix:new_suffix], + old_width=old.width, + new_width=new.width, + ) diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 965b853c34b392e..c169d0191bd8333 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -31,7 +31,6 @@ import sys import code import warnings -import errno from .readline import _get_reader, multiline_input, append_history_file @@ -125,7 +124,7 @@ def maybe_run_command(statement: str) -> bool: command = REPL_COMMANDS[statement] if callable(command): # Make sure that history does not change because of commands - with reader.suspend_history(): + with reader.suspend_history(), reader.suspend_colorization(): command() return True return False @@ -158,10 +157,11 @@ def maybe_run_command(statement: str) -> bool: input_n += 1 except KeyboardInterrupt: r = _get_reader() + r.cmpltn_reset() if r.input_trans is r.isearch_trans: r.do_cmd(("isearch-end", [""])) r.pos = len(r.get_unicode()) - r.dirty = True + r.invalidate_full() r.refresh() console.write("\nKeyboardInterrupt\n") console.resetbuffer() diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py index 943ee12f964b296..395867805196a5c 100644 --- a/Lib/_pyrepl/trace.py +++ b/Lib/_pyrepl/trace.py @@ -32,3 +32,9 @@ def trace(line: str, *k: object, **kw: object) -> None: line = line.format(*k, **kw) trace_file.write(line + "\n") trace_file.flush() + + +def trace_text(text: str, limit: int = 60) -> str: + if len(text) > limit: + text = text[:limit] + "..." + return repr(text) diff --git a/Lib/_pyrepl/types.py b/Lib/_pyrepl/types.py index c5b7ebc1a406bdf..919763158eb123a 100644 --- a/Lib/_pyrepl/types.py +++ b/Lib/_pyrepl/types.py @@ -4,7 +4,13 @@ type SimpleContextManager = Iterator[None] type KeySpec = str # like r"\C-c" type CommandName = str # like "interrupt" -type EventTuple = tuple[CommandName, str] +type EventData = list[str] +type EventTuple = tuple[CommandName, EventData] +type CursorXY = tuple[int, int] +type Dimensions = tuple[int, int] +type ScreenInfoRow = tuple[int, list[int]] +type Keymap = tuple[tuple[KeySpec, CommandName], ...] type Completer = Callable[[str, int], str | None] type CharBuffer = list[str] type CharWidths = list[int] +type CompletionAction = tuple[str, Callable[[], str | None]] diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index a7e49923191c073..9a4f8e0c623abfa 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -31,14 +31,25 @@ import time import types import platform +from collections.abc import Callable +from dataclasses import dataclass from fcntl import ioctl +from typing import TYPE_CHECKING, overload from . import terminfo from .console import Console, Event -from .fancy_termios import tcgetattr, tcsetattr -from .trace import trace +from .fancy_termios import tcgetattr, tcsetattr, TermState +from .render import ( + EMPTY_RENDER_LINE, + LineUpdate, + RenderLine, + RenderedScreen, + requires_cursor_resync, + diff_render_lines, + render_cells, +) +from .trace import trace, trace_text from .unix_eventqueue import EventQueue -from .utils import wlen # declare posix optional to allow None assignment on other platforms posix: types.ModuleType | None @@ -47,20 +58,21 @@ except ImportError: posix = None -TYPE_CHECKING = False - # types if TYPE_CHECKING: - from typing import IO, Literal, overload -else: - overload = lambda func: None + from typing import AbstractSet, IO, Literal + +type _MoveFunc = Callable[[int, int], None] +type _PendingWrite = tuple[str | bytes, bool] class InvalidTerminal(RuntimeError): - pass + def __init__(self, message: str) -> None: + super().__init__(errno.EIO, message) _error = (termios.error, InvalidTerminal) +_error_codes_to_ignore = frozenset([errno.EIO, errno.ENXIO, errno.EPERM]) SIGWINCH_EVENT = "repaint" @@ -125,18 +137,59 @@ def __init__(self): def register(self, fd, flag): self.fd = fd + # note: The 'timeout' argument is received as *milliseconds* def poll(self, timeout: float | None = None) -> list[int]: if timeout is None: r, w, e = select.select([self.fd], [], []) else: - r, w, e = select.select([self.fd], [], [], timeout/1000) + r, w, e = select.select([self.fd], [], [], timeout / 1000) return r poll = MinimalPoll # type: ignore[assignment] +@dataclass(frozen=True, slots=True) +class UnixRefreshPlan: + """Instructions for updating the terminal after a screen change. + + After the user types ``e`` to complete ``name``:: + + Before: >>> def greet(nam|): + ▲ + LineUpdate here: insert_char "e" + + After: >>> def greet(name|): + ▲ + + Only the changed cells are sent to the terminal; unchanged rows + are skipped entirely. + """ + + grow_lines: int + """Number of blank lines to append at the bottom to accommodate new content.""" + use_tall_mode: bool + """Use absolute cursor addressing via ``cup`` instead of relative moves. + Activated when content exceeds one screen height.""" + offset: int + """Vertical scroll offset: the buffer row displayed at the top of the terminal window.""" + reverse_scroll: int + """Number of lines to scroll backwards (content moves down).""" + forward_scroll: int + """Number of lines to scroll forwards (content moves up).""" + line_updates: tuple[LineUpdate, ...] + cleared_lines: tuple[int, ...] + """Row indices to erase (old content with no replacement).""" + rendered_screen: RenderedScreen + cursor: tuple[int, int] + + class UnixConsole(Console): + __buffer: list[_PendingWrite] + __gone_tall: bool + __move: _MoveFunc + __offset: int + def __init__( self, f_in: IO[bytes] | int = 0, @@ -159,9 +212,20 @@ def __init__( self.pollob.register(self.input_fd, select.POLLIN) self.terminfo = terminfo.TermInfo(term or None) self.term = term + self.is_apple_terminal = ( + platform.system() == "Darwin" + and os.getenv("TERM_PROGRAM") == "Apple_Terminal" + ) + + try: + self.__input_fd_set(tcgetattr(self.input_fd), ignore=frozenset()) + except _error as e: + raise RuntimeError(f"termios failure ({e.args[1]})") @overload - def _my_getstr(cap: str, optional: Literal[False] = False) -> bytes: ... + def _my_getstr( + cap: str, optional: Literal[False] = False + ) -> bytes: ... @overload def _my_getstr(cap: str, optional: bool) -> bytes | None: ... @@ -201,8 +265,10 @@ def _my_getstr(cap: str, optional: bool = False) -> bytes | None: self.__setup_movement() - self.event_queue = EventQueue(self.input_fd, self.encoding, self.terminfo) - self.cursor_visible = 1 + self.event_queue = EventQueue( + self.input_fd, self.encoding, self.terminfo + ) + self.cursor_visible = True signal.signal(signal.SIGCONT, self._sigcont_handler) @@ -213,7 +279,6 @@ def _sigcont_handler(self, signum, frame): def __read(self, n: int) -> bytes: return os.read(self.input_fd, n) - def change_encoding(self, encoding: str) -> None: """ Change the encoding used for I/O operations. @@ -223,33 +288,50 @@ def change_encoding(self, encoding: str) -> None: """ self.encoding = encoding - def refresh(self, screen, c_xy): + def refresh(self, rendered_screen: RenderedScreen) -> None: """ Refresh the console screen. Parameters: - - screen (list): List of strings representing the screen contents. - - c_xy (tuple): Cursor position (x, y) on the screen. + - rendered_screen: Structured rendered screen contents and cursor. """ + c_xy = rendered_screen.cursor + trace( + "unix.refresh start cursor={cursor} lines={lines} prev_lines={prev_lines} " + "offset={offset} posxy={posxy}", + cursor=c_xy, + lines=len(rendered_screen.composed_lines), + prev_lines=len(self._rendered_screen.composed_lines), + offset=self.__offset, + posxy=self.posxy, + ) + plan = self.__plan_refresh(rendered_screen, c_xy) + self.__apply_refresh_plan(plan) + + def __plan_refresh( + self, + rendered_screen: RenderedScreen, + c_xy: tuple[int, int], + ) -> UnixRefreshPlan: cx, cy = c_xy + height = self.height + old_offset = offset = self.__offset + prev_composed = self._rendered_screen.composed_lines + previous_lines = list(prev_composed) + next_lines = list(rendered_screen.composed_lines) + line_count = len(next_lines) + + grow_lines = 0 if not self.__gone_tall: - while len(self.screen) < min(len(screen), self.height): - self.__hide_cursor() - self.__move(0, len(self.screen) - 1) - self.__write("\n") - self.posxy = 0, len(self.screen) - self.screen.append("") - else: - while len(self.screen) < len(screen): - self.screen.append("") + grow_lines = max( + min(line_count, height) - len(prev_composed), + 0, + ) + previous_lines.extend([EMPTY_RENDER_LINE] * grow_lines) + elif len(previous_lines) < line_count: + previous_lines.extend([EMPTY_RENDER_LINE] * (line_count - len(previous_lines))) - if len(screen) > self.height: - self.__gone_tall = 1 - self.__move = self.__move_tall - - px, py = self.posxy - old_offset = offset = self.__offset - height = self.height + use_tall_mode = self.__gone_tall or line_count > height # we make sure the cursor is on the screen, and that we're # using all of the screen if we can @@ -257,56 +339,115 @@ def refresh(self, screen, c_xy): offset = cy elif cy >= offset + height: offset = cy - height + 1 - elif offset > 0 and len(screen) < offset + height: - offset = max(len(screen) - height, 0) - screen.append("") + elif offset > 0 and line_count < offset + height: + offset = max(line_count - height, 0) + next_lines.append(EMPTY_RENDER_LINE) - oldscr = self.screen[old_offset : old_offset + height] - newscr = screen[offset : offset + height] + oldscr = previous_lines[old_offset : old_offset + height] + newscr = next_lines[offset : offset + height] - # use hardware scrolling if we have it. + reverse_scroll = 0 + forward_scroll = 0 if old_offset > offset and self._ri: + reverse_scroll = old_offset - offset + for _ in range(reverse_scroll): + if oldscr: + oldscr.pop(-1) + oldscr.insert(0, EMPTY_RENDER_LINE) + elif old_offset < offset and self._ind: + forward_scroll = offset - old_offset + for _ in range(forward_scroll): + if oldscr: + oldscr.pop(0) + oldscr.append(EMPTY_RENDER_LINE) + + line_updates: list[LineUpdate] = [] + px, _ = self.posxy + for y, oldline, newline in zip(range(offset, offset + height), oldscr, newscr): + update = self.__plan_changed_line(y, oldline, newline, px) + if update is not None: + line_updates.append(update) + + cleared_lines = tuple(range(offset + len(newscr), offset + len(oldscr))) + console_rendered_screen = RenderedScreen(tuple(next_lines), c_xy) + trace( + "unix.refresh plan grow={grow} tall={tall} offset={offset} " + "reverse_scroll={reverse_scroll} forward_scroll={forward_scroll} " + "updates={updates} clears={clears}", + grow=grow_lines, + tall=use_tall_mode, + offset=offset, + reverse_scroll=reverse_scroll, + forward_scroll=forward_scroll, + updates=len(line_updates), + clears=len(cleared_lines), + ) + return UnixRefreshPlan( + grow_lines=grow_lines, + use_tall_mode=use_tall_mode, + offset=offset, + reverse_scroll=reverse_scroll, + forward_scroll=forward_scroll, + line_updates=tuple(line_updates), + cleared_lines=cleared_lines, + rendered_screen=console_rendered_screen, + cursor=(cx, cy), + ) + + def __apply_refresh_plan(self, plan: UnixRefreshPlan) -> None: + cx, cy = plan.cursor + trace( + "unix.refresh apply cursor={cursor} updates={updates} clears={clears}", + cursor=plan.cursor, + updates=len(plan.line_updates), + clears=len(plan.cleared_lines), + ) + visual_style = self.begin_redraw_visualization() + screen_line_count = len(self._rendered_screen.composed_lines) + + for _ in range(plan.grow_lines): + self.__hide_cursor() + if screen_line_count: + self.__move(0, screen_line_count - 1) + self.__write("\n") + self.posxy = 0, screen_line_count + screen_line_count += 1 + + if plan.use_tall_mode and not self.__gone_tall: + self.__gone_tall = True + self.__move = self.__move_tall + + old_offset = self.__offset + if plan.reverse_scroll: self.__hide_cursor() self.__write_code(self._cup, 0, 0) self.posxy = 0, old_offset - for i in range(old_offset - offset): + for _ in range(plan.reverse_scroll): self.__write_code(self._ri) - oldscr.pop(-1) - oldscr.insert(0, "") - elif old_offset < offset and self._ind: + elif plan.forward_scroll: self.__hide_cursor() self.__write_code(self._cup, self.height - 1, 0) self.posxy = 0, old_offset + self.height - 1 - for i in range(offset - old_offset): + for _ in range(plan.forward_scroll): self.__write_code(self._ind) - oldscr.pop(0) - oldscr.append("") - self.__offset = offset + self.__offset = plan.offset - for ( - y, - oldline, - newline, - ) in zip(range(offset, offset + height), oldscr, newscr): - if oldline != newline: - self.__write_changed_line(y, oldline, newline, px) + for update in plan.line_updates: + self.__apply_line_update(update, visual_style) - y = len(newscr) - while y < len(oldscr): + for y in plan.cleared_lines: self.__hide_cursor() self.__move(0, y) self.posxy = 0, y self.__write_code(self._el) - y += 1 self.__show_cursor() - - self.screen = screen.copy() self.move_cursor(cx, cy) self.flushoutput() + self.sync_rendered_screen(plan.rendered_screen, self.posxy) - def move_cursor(self, x, y): + def move_cursor(self, x: int, y: int) -> None: """ Move the cursor to the specified position on the screen. @@ -315,16 +456,27 @@ def move_cursor(self, x, y): - y (int): Y coordinate. """ if y < self.__offset or y >= self.__offset + self.height: - self.event_queue.insert(Event("scroll", None)) + trace( + "unix.move_cursor offscreen x={x} y={y} offset={offset} height={height}", + x=x, + y=y, + offset=self.__offset, + height=self.height, + ) + self.event_queue.insert(Event("scroll", "")) else: + trace("unix.move_cursor x={x} y={y}", x=x, y=y) self.__move(x, y) self.posxy = x, y self.flushoutput() - def prepare(self): + def prepare(self) -> None: """ Prepare the console for input/output operations. """ + trace("unix.prepare") + self.__buffer = [] + self.__svtermstate = tcgetattr(self.input_fd) raw = self.__svtermstate.copy() raw.iflag &= ~(termios.INPCK | termios.ISTRIP | termios.IXON) @@ -334,23 +486,22 @@ def prepare(self): raw.iflag |= termios.BRKINT raw.lflag &= ~(termios.ICANON | termios.ECHO | termios.IEXTEN) raw.lflag |= termios.ISIG - raw.cc[termios.VMIN] = 1 - raw.cc[termios.VTIME] = 0 - tcsetattr(self.input_fd, termios.TCSADRAIN, raw) + raw.cc[termios.VMIN] = b"\x01" + raw.cc[termios.VTIME] = b"\x00" + self.__input_fd_set(raw) - # In macOS terminal we need to deactivate line wrap via ANSI escape code - if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + # Apple Terminal will re-wrap lines for us unless we preempt the + # damage. + if self.is_apple_terminal: os.write(self.output_fd, b"\033[?7l") - self.screen = [] self.height, self.width = self.getheightwidth() - self.__buffer = [] - self.posxy = 0, 0 - self.__gone_tall = 0 + self.__gone_tall = False self.__move = self.__move_short self.__offset = 0 + self.sync_rendered_screen(RenderedScreen.empty(), self.posxy) self.__maybe_write_code(self._smkx) @@ -361,20 +512,26 @@ def prepare(self): self.__enable_bracketed_paste() - def restore(self): + def restore(self) -> None: """ Restore the console to the default state """ + trace("unix.restore") self.__disable_bracketed_paste() self.__maybe_write_code(self._rmkx) self.flushoutput() - tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) + self.__input_fd_set(self.__svtermstate) - if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + if self.is_apple_terminal: os.write(self.output_fd, b"\033[?7h") if hasattr(self, "old_sigwinch"): - signal.signal(signal.SIGWINCH, self.old_sigwinch) + try: + signal.signal(signal.SIGWINCH, self.old_sigwinch) + except ValueError as e: + import threading + if threading.current_thread() is threading.main_thread(): + raise e del self.old_sigwinch def push_char(self, char: int | bytes) -> None: @@ -407,6 +564,8 @@ def get_event(self, block: bool = True) -> Event | None: return self.event_queue.get() else: continue + elif err.errno == errno.EIO: + raise SystemExit(errno.EIO) else: raise else: @@ -422,7 +581,7 @@ def wait(self, timeout: float | None = None) -> bool: or bool(self.pollob.poll(timeout)) ) - def set_cursor_vis(self, visible): + def set_cursor_vis(self, visible: bool) -> None: """ Set the visibility of the cursor. @@ -490,8 +649,9 @@ def finish(self): """ Finish console operations and flush the output buffer. """ - y = len(self.screen) - 1 - while y >= 0 and not self.screen[y]: + rendered_lines = self._rendered_screen.composed_lines + y = len(rendered_lines) - 1 + while y >= 0 and not rendered_lines[y].text: y -= 1 self.__move(0, min(y, self.height + self.__offset - 1)) self.__write("\n\r") @@ -518,7 +678,7 @@ def getpending(self): while not self.event_queue.empty(): e2 = self.event_queue.get() e.data += e2.data - e.raw += e.raw + e.raw += e2.raw amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0] trace("getpending({a})", a=amount) @@ -542,7 +702,7 @@ def getpending(self): while not self.event_queue.empty(): e2 = self.event_queue.get() e.data += e2.data - e.raw += e.raw + e.raw += e2.raw amount = 10000 raw = self.__read(amount) @@ -555,11 +715,12 @@ def clear(self): """ Clear the console screen. """ + trace("unix.clear") self.__write_code(self._clear) - self.__gone_tall = 1 + self.__gone_tall = True self.__move = self.__move_tall self.posxy = 0, 0 - self.screen = [] + self.sync_rendered_screen(RenderedScreen.empty(), self.posxy) @property def input_hook(self): @@ -610,98 +771,178 @@ def __setup_movement(self): self.__move = self.__move_short - def __write_changed_line(self, y, oldline, newline, px_coord): - # this is frustrating; there's no reason to test (say) - # self.dch1 inside the loop -- but alternative ways of - # structuring this function are equally painful (I'm trying to - # avoid writing code generators these days...) - minlen = min(wlen(oldline), wlen(newline)) - x_pos = 0 - x_coord = 0 - - px_pos = 0 - j = 0 - for c in oldline: - if j >= px_coord: - break - j += wlen(c) - px_pos += 1 - - # reuse the oldline as much as possible, but stop as soon as we - # encounter an ESCAPE, because it might be the start of an escape - # sequence - while ( - x_coord < minlen - and oldline[x_pos] == newline[x_pos] - and newline[x_pos] != "\x1b" - ): - x_coord += wlen(newline[x_pos]) - x_pos += 1 + @staticmethod + def __cell_index_from_x(line: RenderLine, x_coord: int) -> int: + width = 0 + index = 0 + while index < len(line.cells) and width < x_coord: + width += line.cells[index].width + index += 1 + return index + + def __plan_changed_line( + self, + y: int, + oldline: RenderLine, + newline: RenderLine, + px_coord: int, + ) -> LineUpdate | None: + # NOTE: The shared replace_char / replace_span / rewrite_suffix logic + # is duplicated in WindowsConsole.__plan_changed_line. Keep changes to + # these common cases synchronised between the two files. Yes, this is + # duplicated on purpose; the two backends agree just enough to make a + # shared helper a trap. Unix-only cases (insert_char, delete_then_insert) + # rely on terminal capabilities (ich1/dch1) that are unavailable on + # Windows. + diff = diff_render_lines(oldline, newline) + if diff is None: + return None - # if we need to insert a single character right after the first detected change - if oldline[x_pos:] == newline[x_pos + 1 :] and self.ich1: + start_cell = diff.start_cell + start_x = diff.start_x + + if ( + self.ich1 + and not diff.old_cells + and (visible_new_cells := tuple( + cell for cell in diff.new_cells if cell.width + )) + and len(visible_new_cells) == 1 + and all(cell.width == 0 for cell in diff.new_cells[1:]) + and oldline.cells[start_cell:] == newline.cells[start_cell + 1 :] + ): + px_cell = self.__cell_index_from_x(oldline, px_coord) if ( y == self.posxy[1] - and x_coord > self.posxy[0] - and oldline[px_pos:x_pos] == newline[px_pos + 1 : x_pos + 1] + and start_x > self.posxy[0] + and oldline.cells[px_cell:start_cell] + == newline.cells[px_cell + 1 : start_cell + 1] ): - x_pos = px_pos - x_coord = px_coord - character_width = wlen(newline[x_pos]) - self.__move(x_coord, y) - self.__write_code(self.ich1) - self.__write(newline[x_pos]) - self.posxy = x_coord + character_width, y - - # if it's a single character change in the middle of the line - elif ( - x_coord < minlen - and oldline[x_pos + 1 :] == newline[x_pos + 1 :] - and wlen(oldline[x_pos]) == wlen(newline[x_pos]) + start_cell = px_cell + start_x = px_coord + planned_cells = diff.new_cells + changed_cell = visible_new_cells[0] + return LineUpdate( + kind="insert_char", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=planned_cells, + char_width=changed_cell.width, + reset_to_margin=requires_cursor_resync(planned_cells), + ) + + if ( + len(diff.old_cells) == 1 + and len(diff.new_cells) == 1 + and diff.old_cells[0].width == diff.new_cells[0].width ): - character_width = wlen(newline[x_pos]) - self.__move(x_coord, y) - self.__write(newline[x_pos]) - self.posxy = x_coord + character_width, y - - # if this is the last character to fit in the line and we edit in the middle of the line - elif ( + planned_cells = diff.new_cells + changed_cell = planned_cells[0] + return LineUpdate( + kind="replace_char", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=planned_cells, + char_width=changed_cell.width, + reset_to_margin=requires_cursor_resync(planned_cells), + ) + + if diff.old_changed_width == diff.new_changed_width: + planned_cells = diff.new_cells + return LineUpdate( + kind="replace_span", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=planned_cells, + char_width=diff.new_changed_width, + reset_to_margin=requires_cursor_resync(planned_cells), + ) + + if ( self.dch1 and self.ich1 - and wlen(newline) == self.width - and x_coord < wlen(newline) - 2 - and newline[x_pos + 1 : -1] == oldline[x_pos:-2] + and newline.width == self.width + and start_x < newline.width - 2 + and newline.cells[start_cell + 1 : -1] == oldline.cells[start_cell:-2] ): + planned_cells = (newline.cells[start_cell],) + changed_cell = planned_cells[0] + return LineUpdate( + kind="delete_then_insert", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=planned_cells, + char_width=changed_cell.width, + reset_to_margin=requires_cursor_resync(planned_cells), + ) + + suffix_cells = newline.cells[start_cell:] + return LineUpdate( + kind="rewrite_suffix", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=suffix_cells, + char_width=sum(cell.width for cell in suffix_cells), + clear_eol=oldline.width > newline.width, + reset_to_margin=requires_cursor_resync(suffix_cells), + ) + + def __apply_line_update( + self, + update: LineUpdate, + visual_style: str | None = None, + ) -> None: + text = render_cells(update.cells, visual_style) if visual_style else update.text + trace( + "unix.refresh update kind={kind} y={y} x={x} text={text} " + "clear_eol={clear_eol} reset_to_margin={reset}", + kind=update.kind, + y=update.y, + x=update.start_x, + text=trace_text(text), + clear_eol=update.clear_eol, + reset=update.reset_to_margin, + ) + if update.kind == "insert_char": + self.__move(update.start_x, update.y) + self.__write_code(self.ich1) + self.__write(text) + self.posxy = update.start_x + update.char_width, update.y + elif update.kind in {"replace_char", "replace_span"}: + self.__move(update.start_x, update.y) + self.__write(text) + self.posxy = update.start_x + update.char_width, update.y + elif update.kind == "delete_then_insert": self.__hide_cursor() - self.__move(self.width - 2, y) - self.posxy = self.width - 2, y + self.__move(self.width - 2, update.y) + self.posxy = self.width - 2, update.y self.__write_code(self.dch1) - - character_width = wlen(newline[x_pos]) - self.__move(x_coord, y) + self.__move(update.start_x, update.y) self.__write_code(self.ich1) - self.__write(newline[x_pos]) - self.posxy = character_width + 1, y - + self.__write(text) + self.posxy = update.start_x + update.char_width, update.y else: self.__hide_cursor() - self.__move(x_coord, y) - if wlen(oldline) > wlen(newline): + self.__move(update.start_x, update.y) + if update.clear_eol: self.__write_code(self._el) - self.__write(newline[x_pos:]) - self.posxy = wlen(newline), y + self.__write(text) + self.posxy = update.start_x + update.char_width, update.y - if "\x1b" in newline: - # ANSI escape characters are present, so we can't assume - # anything about the position of the cursor. Moving the cursor - # to the left margin should work to get to a known position. - self.move_cursor(0, y) + if update.reset_to_margin: + # Non-SGR terminal controls can affect the cursor position. + self.move_cursor(0, update.y) def __write(self, text): - self.__buffer.append((text, 0)) + self.__buffer.append((text, False)) def __write_code(self, fmt, *args): - self.__buffer.append((terminfo.tparm(fmt, *args), 1)) + self.__buffer.append((terminfo.tparm(fmt, *args), True)) def __maybe_write_code(self, fmt, *args): if fmt: @@ -752,30 +993,38 @@ def __move_tall(self, x, y): self.__write_code(self._cup, y - self.__offset, x) def __sigwinch(self, signum, frame): - self.height, self.width = self.getheightwidth() - self.event_queue.insert(Event("resize", None)) + self.event_queue.insert(Event("resize", "")) def __hide_cursor(self): if self.cursor_visible: self.__maybe_write_code(self._civis) - self.cursor_visible = 0 + self.cursor_visible = False def __show_cursor(self): if not self.cursor_visible: self.__maybe_write_code(self._cnorm) - self.cursor_visible = 1 + self.cursor_visible = True def repaint(self): + composed = self._rendered_screen.composed_lines + trace( + "unix.repaint gone_tall={gone_tall} screen_lines={lines} offset={offset}", + gone_tall=self.__gone_tall, + lines=len(composed), + offset=self.__offset, + ) if not self.__gone_tall: self.posxy = 0, self.posxy[1] self.__write("\r") - ns = len(self.screen) * ["\000" * self.width] - self.screen = ns + ns = len(composed) * ["\000" * self.width] else: self.posxy = 0, self.__offset self.__move(0, self.__offset) ns = self.height * ["\000" * self.width] - self.screen = ns + self.sync_rendered_screen( + RenderedScreen.from_screen_lines(ns, self.posxy), + self.posxy, + ) def __tputs(self, fmt, prog=delayprog): """A Python implementation of the curses tputs function; the @@ -785,7 +1034,7 @@ def __tputs(self, fmt, prog=delayprog): will never do anyone any good.""" # using .get() means that things will blow up # only if the bps is actually needed (which I'm - # betting is pretty unlkely) + # betting is pretty unlikely) bps = ratedict.get(self.__svtermstate.ospeed) while True: m = prog.search(fmt) @@ -803,3 +1052,17 @@ def __tputs(self, fmt, prog=delayprog): os.write(self.output_fd, self._pad * nchars) else: time.sleep(float(delay) / 1000.0) + + def __input_fd_set( + self, + state: TermState, + ignore: AbstractSet[int] = _error_codes_to_ignore, + ) -> bool: + try: + tcsetattr(self.input_fd, termios.TCSADRAIN, state) + except termios.error as te: + if te.args[0] not in ignore: + raise + return False + else: + return True diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index fd788c8429e15b2..230dae35af665ab 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -9,6 +9,7 @@ import _colorize from collections import deque +from dataclasses import dataclass from io import StringIO from tokenize import TokenInfo as TI from typing import Iterable, Iterator, Match, NamedTuple, Self @@ -16,11 +17,13 @@ from .types import CharBuffer, CharWidths from .trace import trace + ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") ZERO_WIDTH_BRACKET = re.compile(r"\x01.*?\x02") ZERO_WIDTH_TRANS = str.maketrans({"\x01": "", "\x02": ""}) -IDENTIFIERS_AFTER = {"def", "class"} -BUILTINS = {str(name) for name in dir(builtins) if not name.startswith('_')} +IDENTIFIERS_AFTER = frozenset({"def", "class"}) +KEYWORD_CONSTANTS = frozenset({"True", "False", "None"}) +BUILTINS = frozenset({str(name) for name in dir(builtins) if not name.startswith('_')}) def THEME(**kwargs): @@ -58,10 +61,31 @@ class ColorSpan(NamedTuple): tag: str +class StyledChar(NamedTuple): + text: str + width: int + tag: str | None = None + + +def _ascii_control_repr(c: str) -> str | None: + code = ord(c) + if code < 32: + return "^" + chr(code + 64) + if code == 127: + return "^?" + return None + + @functools.cache def str_width(c: str) -> int: if ord(c) < 128: return 1 + # gh-139246 for zero-width joiner and combining characters + if unicodedata.combining(c): + return 0 + category = unicodedata.category(c) + if category == "Cf" and c != "\u00ad": + return 0 w = unicodedata.east_asian_width(c) if w in ("N", "Na", "H", "A"): return 1 @@ -197,8 +221,11 @@ def gen_colors_from_token_stream( span = Span.from_token(token, line_lengths) yield ColorSpan(span, "definition") elif keyword.iskeyword(token.string): + span_cls = "keyword" + if token.string in KEYWORD_CONSTANTS: + span_cls = "keyword_constant" span = Span.from_token(token, line_lengths) - yield ColorSpan(span, "keyword") + yield ColorSpan(span, span_cls) if token.string in IDENTIFIERS_AFTER: is_def_name = True elif ( @@ -208,13 +235,16 @@ def gen_colors_from_token_stream( ): span = Span.from_token(token, line_lengths) yield ColorSpan(span, "soft_keyword") - elif token.string in BUILTINS: + elif ( + token.string in BUILTINS + and not (prev_token and prev_token.exact_type == T.DOT) + ): span = Span.from_token(token, line_lengths) yield ColorSpan(span, "builtin") -keyword_first_sets_match = {"False", "None", "True", "await", "lambda", "not"} -keyword_first_sets_case = {"False", "None", "True"} +keyword_first_sets_match = frozenset({"False", "None", "True", "await", "lambda", "not"}) +keyword_first_sets_case = frozenset({"False", "None", "True"}) def is_soft_keyword_used(*tokens: TI | None) -> bool: @@ -229,7 +259,7 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), TI(string="match"), TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) - | TI(T.OP, string="(" | "*" | "[" | "{" | "~" | "...") + | TI(T.OP, string="(" | "*" | "-" | "+" | "[" | "{" | "~" | "...") ): return True case ( @@ -244,7 +274,7 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), TI(string="case"), TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) - | TI(T.OP, string="(" | "*" | "-" | "[" | "{") + | TI(T.OP, string="(" | "*" | "-" | "+" | "[" | "{") ): return True case ( @@ -257,10 +287,77 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: return True case (TI(string="case"), TI(string="_"), TI(string=":")): return True + case ( + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), + TI(string="type"), + TI(T.NAME, string=s) + ): + return not keyword.iskeyword(s) + case ( + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":" | ";"), + TI(string="lazy"), + TI(string="import") | TI(string="from") + ): + return True case _: return False +def iter_display_chars( + buffer: str, + colors: list[ColorSpan] | None = None, + start_index: int = 0, +) -> Iterator[StyledChar]: + """Yield visible display characters with widths and semantic color tags. + + Note: ``colors`` is consumed in place as spans are processed -- callers + that split a buffer across multiple calls rely on this mutation to track + which spans have already been handled. + """ + + if not buffer: + return + + color_idx = 0 + if colors: + while color_idx < len(colors) and colors[color_idx].span.end < start_index: + color_idx += 1 + + active_tag = None + if colors and color_idx < len(colors) and colors[color_idx].span.start < start_index: + active_tag = colors[color_idx].tag + + for i, c in enumerate(buffer, start_index): + if colors and color_idx < len(colors) and colors[color_idx].span.start == i: + active_tag = colors[color_idx].tag + + if control := _ascii_control_repr(c): + text = control + width = len(control) + elif ord(c) < 128: + text = c + width = 1 + elif unicodedata.category(c).startswith("C"): + text = r"\u%04x" % ord(c) + width = len(text) + else: + text = c + width = str_width(c) + + yield StyledChar(text, width, active_tag) + + if colors and color_idx < len(colors) and colors[color_idx].span.end == i: + color_idx += 1 + active_tag = None + # Check if the next span starts at the same position + if color_idx < len(colors) and colors[color_idx].span.start == i: + active_tag = colors[color_idx].tag + + # Remove consumed spans so callers see the mutation + if color_idx > 0 and colors: + del colors[:color_idx] + + def disp_str( buffer: str, colors: list[ColorSpan] | None = None, @@ -296,53 +393,18 @@ def disp_str( (['\x1b[1;34mw', 'h', 'i', 'l', 'e\x1b[0m', ' ', '1', ':'], [1, 1, 1, 1, 1, 1, 1, 1]) """ + styled_chars = list(iter_display_chars(buffer, colors, start_index)) chars: CharBuffer = [] char_widths: CharWidths = [] - - if not buffer: - return chars, char_widths - - while colors and colors[0].span.end < start_index: - # move past irrelevant spans - colors.pop(0) - theme = THEME(force_color=force_color) - pre_color = "" - post_color = "" - if colors and colors[0].span.start < start_index: - # looks like we're continuing a previous color (e.g. a multiline str) - pre_color = theme[colors[0].tag] - for i, c in enumerate(buffer, start_index): - if colors and colors[0].span.start == i: # new color starts now - pre_color = theme[colors[0].tag] - - if c == "\x1a": # CTRL-Z on Windows - chars.append(c) - char_widths.append(2) - elif ord(c) < 128: - chars.append(c) - char_widths.append(1) - elif unicodedata.category(c).startswith("C"): - c = r"\u%04x" % ord(c) - chars.append(c) - char_widths.append(len(c)) - else: - chars.append(c) - char_widths.append(str_width(c)) - - if colors and colors[0].span.end == i: # current color ends now - post_color = theme.reset - colors.pop(0) - - chars[-1] = pre_color + chars[-1] + post_color - pre_color = "" - post_color = "" - - if colors and colors[0].span.start < i and colors[0].span.end > i: - # even though the current color should be continued, reset it for now. - # the next call to `disp_str()` will revive it. - chars[-1] += theme.reset + for index, styled_char in enumerate(styled_chars): + previous_tag = styled_chars[index - 1].tag if index else None + next_tag = styled_chars[index + 1].tag if index + 1 < len(styled_chars) else None + prefix = theme[styled_char.tag] if styled_char.tag and styled_char.tag != previous_tag else "" + suffix = theme.reset if styled_char.tag and styled_char.tag != next_tag else "" + chars.append(prefix + styled_char.text + suffix) + char_widths.append(styled_char.width) return chars, char_widths @@ -360,13 +422,35 @@ def prev_next_window[T]( """ iterator = iter(iterable) - window = deque((None, next(iterator)), maxlen=3) + try: + first = next(iterator) + except StopIteration: + return + window = deque((None, first), maxlen=3) try: for x in iterator: window.append(x) yield tuple(window) - except Exception: - raise finally: window.append(None) yield tuple(window) + + +@dataclass(frozen=True, slots=True) +class StyleRef: + tag: str | None = None # From THEME().syntax, e.g. "keyword", "builtin" + sgr: str = "" + + @classmethod + def from_tag(cls, tag: str, sgr: str = "") -> Self: + return cls(tag=tag, sgr=sgr) + + @classmethod + def from_sgr(cls, sgr: str) -> Self: + if not sgr: + return cls() + return cls(sgr=sgr) + + @property + def is_plain(self) -> bool: + return self.tag is None and not self.sgr diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index c56dcd6d7dd434f..c1f9a19545d35fd 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -25,6 +25,7 @@ import ctypes import types +from dataclasses import dataclass from ctypes.wintypes import ( _COORD, WORD, @@ -37,20 +38,26 @@ SHORT, ) from ctypes import Structure, POINTER, Union +from typing import TYPE_CHECKING from .console import Event, Console -from .trace import trace -from .utils import wlen +from .render import ( + EMPTY_RENDER_LINE, + LineUpdate, + RenderLine, + RenderedScreen, + requires_cursor_resync, + diff_render_lines, + render_cells, +) +from .trace import trace, trace_text from .windows_eventqueue import EventQueue try: - from ctypes import get_last_error, GetLastError, WinDLL, windll, WinError # type: ignore[attr-defined] + from ctypes import get_last_error, WinDLL, windll, WinError # type: ignore[attr-defined] except: # Keep MyPy happy off Windows from ctypes import CDLL as WinDLL, cdll as windll - def GetLastError() -> int: - return 42 - def get_last_error() -> int: return 42 @@ -66,8 +73,6 @@ def __init__(self, err: int | None, descr: str | None = None) -> None: except ImportError: nt = None -TYPE_CHECKING = False - if TYPE_CHECKING: from typing import IO @@ -126,6 +131,17 @@ def __init__(self, err: int | None, descr: str | None = None) -> None: class _error(Exception): pass + +@dataclass(frozen=True, slots=True) +class WindowsRefreshPlan: + grow_lines: int + offset: int + scroll_lines: int + line_updates: tuple[LineUpdate, ...] + cleared_lines: tuple[int, ...] + rendered_screen: RenderedScreen + cursor: tuple[int, int] + def _supports_vt(): try: return nt._supports_virtual_terminal() @@ -149,18 +165,19 @@ def __init__( # Save original console modes so we can recover on cleanup. original_input_mode = DWORD() - GetConsoleMode(InHandle, original_input_mode) + if not GetConsoleMode(InHandle, original_input_mode): + raise WinError(get_last_error()) trace(f'saved original input mode 0x{original_input_mode.value:x}') self.__original_input_mode = original_input_mode.value - SetConsoleMode( + if not SetConsoleMode( OutHandle, ENABLE_WRAP_AT_EOL_OUTPUT | ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING, - ) + ): + raise WinError(get_last_error()) - self.screen: list[str] = [] self.width = 80 self.height = 25 self.__offset = 0 @@ -171,73 +188,124 @@ def __init__( # Console I/O is redirected, fallback... self.out = None - def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None: + def refresh(self, rendered_screen: RenderedScreen) -> None: """ Refresh the console screen. Parameters: - - screen (list): List of strings representing the screen contents. - - c_xy (tuple): Cursor position (x, y) on the screen. + - rendered_screen: Structured rendered screen contents and cursor. """ - cx, cy = c_xy - - while len(self.screen) < min(len(screen), self.height): - self._hide_cursor() - self._move_relative(0, len(self.screen) - 1) - self.__write("\n") - self.posxy = 0, len(self.screen) - self.screen.append("") + c_xy = rendered_screen.cursor + trace( + "windows.refresh start cursor={cursor} lines={lines} prev_lines={prev_lines} " + "offset={offset} posxy={posxy}", + cursor=c_xy, + lines=len(rendered_screen.composed_lines), + prev_lines=len(self._rendered_screen.composed_lines), + offset=self.__offset, + posxy=self.posxy, + ) + plan = self.__plan_refresh(rendered_screen, c_xy) + self.__apply_refresh_plan(plan) - px, py = self.posxy - old_offset = offset = self.__offset + def __plan_refresh( + self, + rendered_screen: RenderedScreen, + c_xy: tuple[int, int], + ) -> WindowsRefreshPlan: + cx, cy = c_xy height = self.height + old_offset = offset = self.__offset + prev_composed = self._rendered_screen.composed_lines + previous_lines = list(prev_composed) + next_lines = list(rendered_screen.composed_lines) + line_count = len(next_lines) + + grow_lines = max( + min(line_count, height) - len(prev_composed), + 0, + ) + previous_lines.extend([EMPTY_RENDER_LINE] * grow_lines) - # we make sure the cursor is on the screen, and that we're - # using all of the screen if we can + scroll_lines = 0 if cy < offset: offset = cy elif cy >= offset + height: offset = cy - height + 1 scroll_lines = offset - old_offset + previous_lines.extend([EMPTY_RENDER_LINE] * scroll_lines) + elif offset > 0 and line_count < offset + height: + offset = max(line_count - height, 0) + next_lines.append(EMPTY_RENDER_LINE) + + oldscr = previous_lines[old_offset : old_offset + height] + newscr = next_lines[offset : offset + height] + + line_updates: list[LineUpdate] = [] + px, _ = self.posxy + for y, oldline, newline in zip(range(offset, offset + height), oldscr, newscr): + update = self.__plan_changed_line(y, oldline, newline, px) + if update is not None: + line_updates.append(update) + + cleared_lines = tuple(range(offset + len(newscr), offset + len(oldscr))) + console_rendered_screen = RenderedScreen(tuple(next_lines), c_xy) + trace( + "windows.refresh plan grow={grow} offset={offset} scroll_lines={scroll_lines} " + "updates={updates} clears={clears}", + grow=grow_lines, + offset=offset, + scroll_lines=scroll_lines, + updates=len(line_updates), + clears=len(cleared_lines), + ) + return WindowsRefreshPlan( + grow_lines=grow_lines, + offset=offset, + scroll_lines=scroll_lines, + line_updates=tuple(line_updates), + cleared_lines=cleared_lines, + rendered_screen=console_rendered_screen, + cursor=(cx, cy), + ) - # Scrolling the buffer as the current input is greater than the visible - # portion of the window. We need to scroll the visible portion and the - # entire history - self._scroll(scroll_lines, self._getscrollbacksize()) - self.posxy = self.posxy[0], self.posxy[1] + scroll_lines - self.__offset += scroll_lines + def __apply_refresh_plan(self, plan: WindowsRefreshPlan) -> None: + cx, cy = plan.cursor + trace( + "windows.refresh apply cursor={cursor} updates={updates} clears={clears}", + cursor=plan.cursor, + updates=len(plan.line_updates), + clears=len(plan.cleared_lines), + ) + visual_style = self.begin_redraw_visualization() + screen_line_count = len(self._rendered_screen.composed_lines) - for i in range(scroll_lines): - self.screen.append("") - elif offset > 0 and len(screen) < offset + height: - offset = max(len(screen) - height, 0) - screen.append("") + for _ in range(plan.grow_lines): + self._hide_cursor() + if screen_line_count: + self._move_relative(0, screen_line_count - 1) + self.__write("\n") + self.posxy = 0, screen_line_count + screen_line_count += 1 - oldscr = self.screen[old_offset : old_offset + height] - newscr = screen[offset : offset + height] + if plan.scroll_lines: + self._scroll(plan.scroll_lines, self._getscrollbacksize()) + self.posxy = self.posxy[0], self.posxy[1] + plan.scroll_lines - self.__offset = offset + self.__offset = plan.offset self._hide_cursor() - for ( - y, - oldline, - newline, - ) in zip(range(offset, offset + height), oldscr, newscr): - if oldline != newline: - self.__write_changed_line(y, oldline, newline, px) - - y = len(newscr) - while y < len(oldscr): + for update in plan.line_updates: + self.__apply_line_update(update, visual_style) + + for y in plan.cleared_lines: self._move_relative(0, y) self.posxy = 0, y self._erase_to_end() - y += 1 self._show_cursor() - - self.screen = screen self.move_cursor(cx, cy) + self.sync_rendered_screen(plan.rendered_screen, self.posxy) @property def input_hook(self): @@ -246,54 +314,98 @@ def input_hook(self): if nt is not None and nt._is_inputhook_installed(): return nt._inputhook - def __write_changed_line( - self, y: int, oldline: str, newline: str, px_coord: int - ) -> None: - # this is frustrating; there's no reason to test (say) - # self.dch1 inside the loop -- but alternative ways of - # structuring this function are equally painful (I'm trying to - # avoid writing code generators these days...) - minlen = min(wlen(oldline), wlen(newline)) - x_pos = 0 - x_coord = 0 - - px_pos = 0 - j = 0 - for c in oldline: - if j >= px_coord: - break - j += wlen(c) - px_pos += 1 - - # reuse the oldline as much as possible, but stop as soon as we - # encounter an ESCAPE, because it might be the start of an escape - # sequence - while ( - x_coord < minlen - and oldline[x_pos] == newline[x_pos] - and newline[x_pos] != "\x1b" + def __plan_changed_line( # keep in sync with UnixConsole.__plan_changed_line + self, + y: int, + oldline: RenderLine, + newline: RenderLine, + px_coord: int, + ) -> LineUpdate | None: + diff = diff_render_lines(oldline, newline) + if diff is None: + return None + + start_cell = diff.start_cell + start_x = diff.start_x + if ( + len(diff.old_cells) == 1 + and len(diff.new_cells) == 1 + and diff.old_cells[0].width == diff.new_cells[0].width ): - x_coord += wlen(newline[x_pos]) - x_pos += 1 + changed_cell = diff.new_cells[0] + # Ctrl-Z (SUB) can reach here via RenderLine.from_rendered_text() + # for prompt/message lines, which bypasses iter_display_chars(). + # On Windows, raw \x1a causes console cursor anomalies, so we + # force a cursor resync when it appears. + return LineUpdate( + kind="replace_char", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=diff.new_cells, + char_width=changed_cell.width, + reset_to_margin=( + requires_cursor_resync(diff.new_cells) + or "\x1a" in changed_cell.text + ), + ) + + if diff.old_changed_width == diff.new_changed_width: + return LineUpdate( + kind="replace_span", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=diff.new_cells, + char_width=diff.new_changed_width, + reset_to_margin=( + requires_cursor_resync(diff.new_cells) + or any("\x1a" in cell.text for cell in diff.new_cells) + ), + ) + + suffix_cells = newline.cells[start_cell:] + return LineUpdate( + kind="rewrite_suffix", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=suffix_cells, + char_width=sum(cell.width for cell in suffix_cells), + clear_eol=oldline.width > newline.width, + reset_to_margin=( + requires_cursor_resync(suffix_cells) + or any("\x1a" in cell.text for cell in suffix_cells) + ), + ) - self._hide_cursor() - self._move_relative(x_coord, y) - if wlen(oldline) > wlen(newline): + def __apply_line_update( + self, + update: LineUpdate, + visual_style: str | None = None, + ) -> None: + text = render_cells(update.cells, visual_style) if visual_style else update.text + trace( + "windows.refresh update kind={kind} y={y} x={x} text={text} " + "clear_eol={clear_eol} reset_to_margin={reset}", + kind=update.kind, + y=update.y, + x=update.start_x, + text=trace_text(text), + clear_eol=update.clear_eol, + reset=update.reset_to_margin, + ) + original_y = self.posxy[1] + self._move_relative(update.start_x, update.y) + if update.clear_eol: self._erase_to_end() - self.__write(newline[x_pos:]) - if wlen(newline) == self.width: - # If we wrapped we want to start at the next line - self._move_relative(0, y + 1) - self.posxy = 0, y + 1 - else: - self.posxy = wlen(newline), y + self.__write(text) + self.posxy = min(update.start_x + update.char_width, self.width - 1), update.y - if "\x1b" in newline or y != self.posxy[1] or '\x1a' in newline: - # ANSI escape characters are present, so we can't assume - # anything about the position of the cursor. Moving the cursor - # to the left margin should work to get to a known position. - self.move_cursor(0, y) + if update.reset_to_margin or update.y != original_y: + # Non-SGR terminal controls or vertical movement require a cursor sync. + self.move_cursor(0, update.y) def _scroll( self, top: int, bottom: int, left: int | None = None, right: int | None = None @@ -312,7 +424,7 @@ def _scroll( if not ScrollConsoleScreenBuffer( OutHandle, scroll_rect, None, destination_origin, fill_info ): - raise WinError(GetLastError()) + raise WinError(get_last_error()) def _hide_cursor(self): self.__write("\x1b[?25l") @@ -334,7 +446,7 @@ def _disable_bracketed_paste(self) -> None: def __write(self, text: str) -> None: if "\x1a" in text: - text = ''.join(["^Z" if x == '\x1a' else x for x in text]) + text = text.replace("\x1a", "^Z") if self.out is not None: self.out.write(text.encode(self.encoding, "replace")) @@ -346,30 +458,32 @@ def __write(self, text: str) -> None: def screen_xy(self) -> tuple[int, int]: info = CONSOLE_SCREEN_BUFFER_INFO() if not GetConsoleScreenBufferInfo(OutHandle, info): - raise WinError(GetLastError()) + raise WinError(get_last_error()) return info.dwCursorPosition.X, info.dwCursorPosition.Y def _erase_to_end(self) -> None: self.__write(ERASE_IN_LINE) def prepare(self) -> None: - trace("prepare") - self.screen = [] + trace("windows.prepare") self.height, self.width = self.getheightwidth() self.posxy = 0, 0 - self.__gone_tall = 0 self.__offset = 0 + self.sync_rendered_screen(RenderedScreen.empty(), self.posxy) if self.__vt_support: - SetConsoleMode(InHandle, self.__original_input_mode | ENABLE_VIRTUAL_TERMINAL_INPUT) + if not SetConsoleMode(InHandle, self.__original_input_mode | ENABLE_VIRTUAL_TERMINAL_INPUT): + raise WinError(get_last_error()) self._enable_bracketed_paste() def restore(self) -> None: + trace("windows.restore") if self.__vt_support: # Recover to original mode before running REPL self._disable_bracketed_paste() - SetConsoleMode(InHandle, self.__original_input_mode) + if not SetConsoleMode(InHandle, self.__original_input_mode): + raise WinError(get_last_error()) def _move_relative(self, x: int, y: int) -> None: """Moves relative to the current posxy""" @@ -390,8 +504,16 @@ def move_cursor(self, x: int, y: int) -> None: raise ValueError(f"Bad cursor position {x}, {y}") if y < self.__offset or y >= self.__offset + self.height: + trace( + "windows.move_cursor offscreen x={x} y={y} offset={offset} height={height}", + x=x, + y=y, + offset=self.__offset, + height=self.height, + ) self.event_queue.insert(Event("scroll", "")) else: + trace("windows.move_cursor x={x} y={y}", x=x, y=y) self._move_relative(x, y) self.posxy = x, y @@ -406,7 +528,7 @@ def getheightwidth(self) -> tuple[int, int]: and width of the terminal window in characters.""" info = CONSOLE_SCREEN_BUFFER_INFO() if not GetConsoleScreenBufferInfo(OutHandle, info): - raise WinError(GetLastError()) + raise WinError(get_last_error()) return ( info.srWindow.Bottom - info.srWindow.Top + 1, info.srWindow.Right - info.srWindow.Left + 1, @@ -415,7 +537,7 @@ def getheightwidth(self) -> tuple[int, int]: def _getscrollbacksize(self) -> int: info = CONSOLE_SCREEN_BUFFER_INFO() if not GetConsoleScreenBufferInfo(OutHandle, info): - raise WinError(GetLastError()) + raise WinError(get_last_error()) return info.srWindow.Bottom # type: ignore[no-any-return] @@ -423,7 +545,7 @@ def _read_input(self) -> INPUT_RECORD | None: rec = INPUT_RECORD() read = DWORD() if not ReadConsoleInput(InHandle, rec, 1, read): - raise WinError(GetLastError()) + raise WinError(get_last_error()) return rec @@ -433,7 +555,7 @@ def _read_input_bulk( rec = (n * INPUT_RECORD)() read = DWORD() if not ReadConsoleInput(InHandle, rec, n, read): - raise WinError(GetLastError()) + raise WinError(get_last_error()) return rec, read.value @@ -512,15 +634,17 @@ def beep(self) -> None: def clear(self) -> None: """Wipe the screen""" + trace("windows.clear") self.__write(CLEAR) self.posxy = 0, 0 - self.screen = [""] + self.sync_rendered_screen(RenderedScreen.empty(), self.posxy) def finish(self) -> None: """Move the cursor to the end of the display and otherwise get ready for end. XXX could be merged with restore? Hmm.""" - y = len(self.screen) - 1 - while y >= 0 and not self.screen[y]: + rendered_lines = self._rendered_screen.composed_lines + y = len(rendered_lines) - 1 + while y >= 0 and not rendered_lines[y].text: y -= 1 self._move_relative(0, min(y, self.height + self.__offset - 1)) self.__write("\r\n") @@ -535,7 +659,7 @@ def flushoutput(self) -> None: def forgetinput(self) -> None: """Forget all pending, but not yet processed input.""" if not FlushConsoleInputBuffer(InHandle): - raise WinError(GetLastError()) + raise WinError(get_last_error()) def getpending(self) -> Event: """Return the characters that have been typed but not yet @@ -562,11 +686,11 @@ def getpending(self) -> Event: # ignore SHIFT_PRESSED and special keys continue if ch == "\r": - ch += "\n" + ch = "\n" e.data += ch return e - def wait(self, timeout: float | None) -> bool: + def wait_for_event(self, timeout: float | None) -> bool: """Wait for an event.""" if timeout is None: timeout = INFINITE @@ -579,7 +703,17 @@ def wait(self, timeout: float | None) -> bool: return False return True + def wait(self, timeout: float | None) -> bool: + """ + Wait for events on the console. + """ + return ( + not self.event_queue.empty() + or self.wait_for_event(timeout) + ) + def repaint(self) -> None: + trace("windows.repaint unsupported") raise NotImplementedError("No repaint support") diff --git a/Lib/_sitebuiltins.py b/Lib/_sitebuiltins.py index c66269a571967f8..5259a36123ea276 100644 --- a/Lib/_sitebuiltins.py +++ b/Lib/_sitebuiltins.py @@ -36,7 +36,7 @@ def __init__(self, name, data, files=(), dirs=()): import os self.__name = name self.__data = data - self.__lines = None + self.__lines = [] self.__filenames = [os.path.join(dir, filename) for dir in dirs for filename in files] @@ -55,7 +55,6 @@ def __setup(self): if not data: data = self.__data self.__lines = data.split('\n') - self.__linecnt = len(self.__lines) def __repr__(self): self.__setup() @@ -65,24 +64,22 @@ def __repr__(self): return "Type %s() to see the full %s text" % ((self.__name,)*2) def __call__(self): - self.__setup() - prompt = 'Hit Return for more, or q (and Return) to quit: ' - lineno = 0 - while 1: + try: + from _pyrepl.pager import get_pager + except ModuleNotFoundError: try: - for i in range(lineno, lineno + self.MAXLINES): - print(self.__lines[i]) - except IndexError: - break - else: - lineno += self.MAXLINES - key = None - while key is None: - key = input(prompt) - if key not in ('', 'q'): - key = None - if key == 'q': - break + from pydoc import get_pager + except ModuleNotFoundError: + def get_pager(): + def _print(text, title=None): + print(text) + return _print + + self.__setup() + + pager = get_pager() + text = "\n".join(self.__lines) + pager(text, title=self.__name) class _Helper(object): diff --git a/Lib/_strptime.py b/Lib/_strptime.py index cdc55e8daaffa6d..746b0907c1d9f4e 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -7,7 +7,7 @@ FUNCTIONS: _getlang -- Figure out what language is being used for the locale - strptime -- Calculates the time struct represented by the passed-in string + _strptime -- Calculates the time struct represented by the passed-in string """ import os @@ -371,7 +371,9 @@ def __init__(self, locale_time=None): # W is set below by using 'U' 'y': r"(?P\d\d)", 'Y': r"(?P\d\d\d\d)", + # See gh-121237: "z" must support colons for backwards compatibility. 'z': r"(?P([+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?)|(?-i:Z))?", + ':z': r"(?P([+-]\d\d:[0-5]\d(:[0-5]\d(\.\d{1,6})?)?)|(?-i:Z))?", 'A': self.__seqToRE(self.locale_time.f_weekday, 'A'), 'a': self.__seqToRE(self.locale_time.a_weekday, 'a'), 'B': self.__seqToRE(_fixmonths(self.locale_time.f_month[1:]), 'B'), @@ -380,7 +382,10 @@ def __init__(self, locale_time=None): 'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone for tz in tz_names), 'Z'), - '%': '%'} + 'n': r'\s*', + 't': r'\s*', + '%': '%', + } if self.locale_time.LC_alt_digits is None: for d in 'dmyCHIMS': mapping['O' + d] = r'(?P<%s>\d\d|\d| \d)' % d @@ -416,6 +421,8 @@ def __init__(self, locale_time=None): mapping['W'] = mapping['U'].replace('U', 'W') base.__init__(mapping) + base.__setitem__('D', self.pattern('%m/%d/%y')) + base.__setitem__('F', self.pattern('%Y-%m-%d')) base.__setitem__('T', self.pattern('%H:%M:%S')) base.__setitem__('R', self.pattern('%H:%M')) base.__setitem__('r', self.pattern(self.locale_time.LC_time_ampm)) @@ -457,28 +464,39 @@ def pattern(self, format): format = re_sub(r'\s+', r'\\s+', format) format = re_sub(r"'", "['\u02bc]", format) # needed for br_FR year_in_format = False - day_of_month_in_format = False + day_d_in_format = False + day_e_in_format = False def repl(m): - format_char = m[1] - match format_char: + directive = m.group()[1:] # exclude `%` symbol + match directive: case 'Y' | 'y' | 'G': nonlocal year_in_format year_in_format = True case 'd': - nonlocal day_of_month_in_format - day_of_month_in_format = True - return self[format_char] - format = re_sub(r'%[-_0^#]*[0-9]*([OE]?\\?.?)', repl, format) - if day_of_month_in_format and not year_in_format: - import warnings - warnings.warn("""\ + nonlocal day_d_in_format + day_d_in_format = True + case 'e': + nonlocal day_e_in_format + day_e_in_format = True + return self[directive] + format = re_sub(r'%[-_0^#]*[0-9]*([OE]?[:\\]?.?)', repl, format) + if not year_in_format: + if day_d_in_format: + raise ValueError( + "Day of month directive '%d' may not be used without " + "a year directive. Parsing dates involving a day of " + "month without a year is ambiguous and fails to parse " + "leap day. Add a year to the input and format. " + "See https://github.com/python/cpython/issues/70647.") + if day_e_in_format: + import warnings + warnings.warn("""\ Parsing dates involving a day of month without a year specified is ambiguous -and fails to parse leap day. The default behavior will change in Python 3.15 -to either always raise an exception or to use a different default year (TBD). -To avoid trouble, add a specific year to the input & format. +and fails to parse leap day. '%e' without a year will become an error in Python 3.17. +To avoid trouble, add a specific year to the input and format. See https://github.com/python/cpython/issues/70647.""", - DeprecationWarning, - skip_file_prefixes=(os.path.dirname(__file__),)) + DeprecationWarning, + skip_file_prefixes=(os.path.dirname(__file__),)) return format def compile(self, format): @@ -514,9 +532,10 @@ def _calc_julian_from_U_or_W(year, week_of_year, day_of_week, week_starts_Mon): def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): - """Return a 2-tuple consisting of a time struct and an int containing - the number of microseconds based on the input string and the - format string.""" + """Return a 3-tuple consisting of a tuple with time components, + an int containing the number of microseconds, and an int + containing the microseconds part of the GMT offset, based on the + input string and the format string.""" for index, arg in enumerate([data_string, format]): if not isinstance(arg, str): @@ -555,8 +574,17 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): raise ValueError("time data %r does not match format %r" % (data_string, format)) if len(data_string) != found.end(): - raise ValueError("unconverted data remains: %s" % - data_string[found.end():]) + rest = data_string[found.end():] + # Specific check for '%:z' directive + if ( + "colon_z" in found.re.groupindex + and found.group("colon_z") is not None + and rest[0] != ":" + ): + raise ValueError( + f"Missing colon in %:z before '{rest}', got '{data_string}'" + ) + raise ValueError("unconverted data remains: %s" % rest) iso_year = year = None month = day = 1 @@ -616,18 +644,18 @@ def parse_int(s): hour = parse_int(found_dict['I']) ampm = found_dict.get('p', '').lower() # If there was no AM/PM indicator, we'll treat this like AM - if ampm in ('', locale_time.am_pm[0]): - # We're in AM so the hour is correct unless we're - # looking at 12 midnight. - # 12 midnight == 12 AM == hour 0 - if hour == 12: - hour = 0 - elif ampm == locale_time.am_pm[1]: + if ampm == locale_time.am_pm[1]: # We're in PM so we need to add 12 to the hour unless # we're looking at 12 noon. # 12 noon == 12 PM == hour 12 if hour != 12: hour += 12 + else: + # We're in AM so the hour is correct unless we're + # looking at 12 midnight. + # 12 midnight == 12 AM == hour 0 + if hour == 12: + hour = 0 elif group_key == 'M': minute = parse_int(found_dict['M']) elif group_key == 'S': @@ -662,8 +690,8 @@ def parse_int(s): week_of_year_start = 0 elif group_key == 'V': iso_week = int(found_dict['V']) - elif group_key == 'z': - z = found_dict['z'] + elif group_key in ('z', 'colon_z'): + z = found_dict[group_key] if z: if z == 'Z': gmtoff = 0 @@ -672,7 +700,7 @@ def parse_int(s): z = z[:3] + z[4:] if len(z) > 5: if z[5] != ':': - msg = f"Inconsistent use of : in {found_dict['z']}" + msg = f"Inconsistent use of : in {found_dict[group_key]}" raise ValueError(msg) z = z[:5] + z[6:] hours = int(z[1:3]) diff --git a/Lib/_threading_local.py b/Lib/_threading_local.py index 0b9e5d3bbf6ef64..2af3885458b54f2 100644 --- a/Lib/_threading_local.py +++ b/Lib/_threading_local.py @@ -57,7 +57,7 @@ def thread_deleted(_, idt=idt): # as soon as the OS-level thread ends instead. local = wrlocal() if local is not None: - dct = local.dicts.pop(idt) + local.dicts.pop(idt) wrlocal = ref(self, local_deleted) wrthread = ref(thread, thread_deleted) thread.__dict__[key] = wrlocal diff --git a/Lib/abc.py b/Lib/abc.py index f8a4e11ce9c3b1e..08f708d1b892085 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -36,11 +36,15 @@ class C(ABC): def my_abstract_classmethod(cls, ...): ... + .. deprecated-removed: 3.3 3.21 + """ __isabstractmethod__ = True def __init__(self, callable): + import warnings + warnings._deprecated('abc.abstractclassmethod', remove=(3, 21)) callable.__isabstractmethod__ = True super().__init__(callable) @@ -56,11 +60,15 @@ class C(ABC): def my_abstract_staticmethod(...): ... + .. deprecated-removed: 3.3 3.21 + """ __isabstractmethod__ = True def __init__(self, callable): + import warnings + warnings._deprecated('abc.abstractstaticmethod', remove=(3, 21)) callable.__isabstractmethod__ = True super().__init__(callable) @@ -76,10 +84,23 @@ class C(ABC): def my_abstract_property(self): ... + .. deprecated-removed: 3.3 3.21 + """ __isabstractmethod__ = True + def __init__( + self, + fget=None, + fset=None, + fdel=None, + doc=None, + ): + import warnings + warnings._deprecated('abc.abstractproperty', remove=(3, 21)) + super().__init__(fget, fset, fdel, doc) + try: from _abc import (get_cache_token, _abc_init, _abc_register, diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index c83a1573ccd3d1e..8204c762cce8a2b 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -47,6 +47,7 @@ class Format(enum.IntEnum): "__cell__", "__owner__", "__stringifier_dict__", + "__resolved_str_cache__", ) @@ -85,12 +86,16 @@ def __init__( # These are always set to None here but may be non-None if a ForwardRef # is created through __class__ assignment on a _Stringifier object. self.__globals__ = None + # This may be either a cell object (for a ForwardRef referring to a single name) + # or a dict mapping cell names to cell objects (for a ForwardRef containing references + # to multiple names). self.__cell__ = None self.__extra_names__ = None # These are initially None but serve as a cache and may be set to a non-None # value later. self.__code__ = None self.__ast_node__ = None + self.__resolved_str_cache__ = None def __init_subclass__(cls, /, *args, **kwds): raise TypeError("Cannot subclass ForwardRef") @@ -110,14 +115,14 @@ def evaluate( """ match format: case Format.STRING: - return self.__forward_arg__ + return self.__resolved_str__ case Format.VALUE: is_forwardref_format = False case Format.FORWARDREF: is_forwardref_format = True case _: raise NotImplementedError(format) - if self.__cell__ is not None: + if isinstance(self.__cell__, types.CellType): try: return self.__cell__.cell_contents except ValueError: @@ -147,34 +152,42 @@ def evaluate( if globals is None: globals = {} + if type_params is None and owner is not None: + type_params = getattr(owner, "__type_params__", None) + if locals is None: locals = {} if isinstance(owner, type): locals.update(vars(owner)) + elif ( + type_params is not None + or isinstance(self.__cell__, dict) + or self.__extra_names__ + ): + # Create a new locals dict if necessary, + # to avoid mutating the argument. + locals = dict(locals) - if type_params is None and owner is not None: - # "Inject" type parameters into the local namespace - # (unless they are shadowed by assignments *in* the local namespace), - # as a way of emulating annotation scopes when calling `eval()` - type_params = getattr(owner, "__type_params__", None) - - # type parameters require some special handling, - # as they exist in their own scope - # but `eval()` does not have a dedicated parameter for that scope. - # For classes, names in type parameter scopes should override - # names in the global scope (which here are called `localns`!), - # but should in turn be overridden by names in the class scope - # (which here are called `globalns`!) + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` if type_params is not None: - globals = dict(globals) - locals = dict(locals) for param in type_params: - param_name = param.__name__ - if not self.__forward_is_class__ or param_name not in globals: - globals[param_name] = param - locals.pop(param_name, None) + locals.setdefault(param.__name__, param) + + # Similar logic can be used for nonlocals, which should not + # override locals. + if isinstance(self.__cell__, dict): + for cell_name, cell in self.__cell__.items(): + try: + cell_value = cell.cell_contents + except ValueError: + pass + else: + locals.setdefault(cell_name, cell_value) + if self.__extra_names__: - locals = {**locals, **self.__extra_names__} + locals.update(self.__extra_names__) arg = self.__forward_arg__ if arg.isidentifier() and not keyword.iskeyword(arg): @@ -195,8 +208,11 @@ def evaluate( except Exception: if not is_forwardref_format: raise + + # All variables, in scoping order, should be checked before + # triggering __missing__ to create a _Stringifier. new_locals = _StringifierDict( - {**builtins.__dict__, **locals}, + {**builtins.__dict__, **globals, **locals}, globals=globals, owner=owner, is_class=self.__forward_is_class__, @@ -207,32 +223,9 @@ def evaluate( except Exception: return self else: - new_locals.transmogrify() + new_locals.transmogrify(self.__cell__) return result - def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): - import typing - import warnings - - if type_params is _sentinel: - typing._deprecation_warning_for_no_type_params_passed( - "typing.ForwardRef._evaluate" - ) - type_params = () - warnings._deprecated( - "ForwardRef._evaluate", - "{name} is a private API and is retained for compatibility, but will be removed" - " in Python 3.16. Use ForwardRef.evaluate() or typing.evaluate_forward_ref() instead.", - remove=(3, 16), - ) - return typing.evaluate_forward_ref( - self, - globals=globalns, - locals=localns, - type_params=type_params, - _recursive_guard=recursive_guard, - ) - @property def __forward_arg__(self): if self.__arg__ is not None: @@ -244,20 +237,31 @@ def __forward_arg__(self): "Attempted to access '__forward_arg__' on an uninitialized ForwardRef" ) + @property + def __resolved_str__(self): + # __forward_arg__ with any names from __extra_names__ replaced + # with the type_repr of the value they represent + if self.__resolved_str_cache__ is None: + resolved_str = self.__forward_arg__ + names = self.__extra_names__ + + if names: + visitor = _ExtraNameFixer(names) + ast_expr = ast.parse(resolved_str, mode="eval").body + node = visitor.visit(ast_expr) + resolved_str = ast.unparse(node) + + self.__resolved_str_cache__ = resolved_str + + return self.__resolved_str_cache__ + @property def __forward_code__(self): if self.__code__ is not None: return self.__code__ arg = self.__forward_arg__ - # If we do `def f(*args: *Ts)`, then we'll have `arg = '*Ts'`. - # Unfortunately, this isn't a valid expression on its own, so we - # do the unpacking manually. - if arg.startswith("*"): - arg_to_compile = f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] - else: - arg_to_compile = arg try: - self.__code__ = compile(arg_to_compile, "", "eval") + self.__code__ = compile(_rewrite_star_unpack(arg), "", "eval") except SyntaxError: raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") return self.__code__ @@ -272,7 +276,13 @@ def __eq__(self, other): # because dictionaries are not hashable. and self.__globals__ is other.__globals__ and self.__forward_is_class__ == other.__forward_is_class__ - and self.__cell__ == other.__cell__ + # Two separate cells are always considered unequal in forward refs. + and ( + {name: id(cell) for name, cell in self.__cell__.items()} + == {name: id(cell) for name, cell in other.__cell__.items()} + if isinstance(self.__cell__, dict) and isinstance(other.__cell__, dict) + else self.__cell__ is other.__cell__ + ) and self.__owner__ == other.__owner__ and ( (tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None) == @@ -286,7 +296,10 @@ def __hash__(self): self.__forward_module__, id(self.__globals__), # dictionaries are not hashable, so hash by identity self.__forward_is_class__, - self.__cell__, + ( # cells are not hashable as well + tuple(sorted([(name, id(cell)) for name, cell in self.__cell__.items()])) + if isinstance(self.__cell__, dict) else id(self.__cell__), + ), self.__owner__, tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None, )) @@ -305,7 +318,7 @@ def __repr__(self): extra.append(", is_class=True") if self.__owner__ is not None: extra.append(f", owner={self.__owner__!r}") - return f"ForwardRef({self.__forward_arg__!r}{''.join(extra)})" + return f"ForwardRef({self.__resolved_str__!r}{''.join(extra)})" _Template = type(t"") @@ -341,6 +354,7 @@ def __init__( self.__cell__ = cell self.__owner__ = owner self.__stringifier_dict__ = stringifier_dict + self.__resolved_str_cache__ = None # Needed for ForwardRef def __convert_to_ast(self, other): if isinstance(other, _Stringifier): @@ -568,32 +582,70 @@ def unary_op(self): del _make_unary_op -def _template_to_ast(template): +def _template_to_ast_constructor(template): + """Convert a `template` instance to a non-literal AST.""" + args = [] + for part in template: + match part: + case str(): + args.append(ast.Constant(value=part)) + case _: + interp = ast.Call( + func=ast.Name(id="Interpolation"), + args=[ + ast.Constant(value=part.value), + ast.Constant(value=part.expression), + ast.Constant(value=part.conversion), + ast.Constant(value=part.format_spec), + ] + ) + args.append(interp) + return ast.Call(func=ast.Name(id="Template"), args=args, keywords=[]) + + +def _template_to_ast_literal(template, parsed): + """Convert a `template` instance to a t-string literal AST.""" values = [] + interp_count = 0 for part in template: match part: case str(): values.append(ast.Constant(value=part)) - # Interpolation, but we don't want to import the string module case _: interp = ast.Interpolation( str=part.expression, - value=ast.parse(part.expression), - conversion=( - ord(part.conversion) - if part.conversion is not None - else -1 - ), - format_spec=( - ast.Constant(value=part.format_spec) - if part.format_spec != "" - else None - ), + value=parsed[interp_count], + conversion=ord(part.conversion) if part.conversion else -1, + format_spec=ast.Constant(value=part.format_spec) + if part.format_spec + else None, ) values.append(interp) + interp_count += 1 return ast.TemplateStr(values=values) +def _template_to_ast(template): + """Make a best-effort conversion of a `template` instance to an AST.""" + # gh-138558: Not all Template instances can be represented as t-string + # literals. Return the most accurate AST we can. See issue for details. + + # If any expr is empty or whitespace only, we cannot convert to a literal. + if any(part.expression.strip() == "" for part in template.interpolations): + return _template_to_ast_constructor(template) + + try: + # Wrap in parens to allow whitespace inside interpolation curly braces + parsed = tuple( + ast.parse(f"({part.expression})", mode="eval").body + for part in template.interpolations + ) + except SyntaxError: + return _template_to_ast_constructor(template) + + return _template_to_ast_literal(template, parsed) + + class _StringifierDict(dict): def __init__(self, namespace, *, globals=None, owner=None, is_class=False, format): super().__init__(namespace) @@ -616,13 +668,15 @@ def __missing__(self, key): self.stringifiers.append(fwdref) return fwdref - def transmogrify(self): + def transmogrify(self, cell_dict): for obj in self.stringifiers: obj.__class__ = ForwardRef obj.__stringifier_dict__ = None # not needed for ForwardRef if isinstance(obj.__ast_node__, str): obj.__arg__ = obj.__ast_node__ obj.__ast_node__ = None + if cell_dict is not None and obj.__cell__ is None: + obj.__cell__ = cell_dict def create_unique_name(self): name = f"__annotationlib_name_{self.next_id}__" @@ -672,9 +726,21 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): # possibly constants if the annotate function uses them directly). We then # convert each of those into a string to get an approximation of the # original source. + + # Attempt to call with VALUE_WITH_FAKE_GLOBALS to check if it is implemented + # See: https://github.com/python/cpython/issues/138764 + # Only fail on NotImplementedError + try: + annotate(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # Both STRING and VALUE_WITH_FAKE_GLOBALS are not implemented: fallback to VALUE + return annotations_to_string(annotate(Format.VALUE)) + except Exception: + pass + globals = _StringifierDict({}, format=format) is_class = isinstance(owner, type) - closure = _build_closure( + closure, _ = _build_closure( annotate, owner, is_class, globals, allow_evaluation=False ) func = types.FunctionType( @@ -718,7 +784,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): is_class=is_class, format=format, ) - closure = _build_closure( + closure, cell_dict = _build_closure( annotate, owner, is_class, globals, allow_evaluation=True ) func = types.FunctionType( @@ -730,10 +796,13 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): ) try: result = func(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # FORWARDREF and VALUE_WITH_FAKE_GLOBALS not supported, fall back to VALUE + return annotate(Format.VALUE) except Exception: pass else: - globals.transmogrify() + globals.transmogrify(cell_dict) return result # Try again, but do not provide any globals. This allows us to return @@ -745,7 +814,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): is_class=is_class, format=format, ) - closure = _build_closure( + closure, cell_dict = _build_closure( annotate, owner, is_class, globals, allow_evaluation=False ) func = types.FunctionType( @@ -756,7 +825,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): kwdefaults=annotate.__kwdefaults__, ) result = func(Format.VALUE_WITH_FAKE_GLOBALS) - globals.transmogrify() + globals.transmogrify(cell_dict) if _is_evaluate: if isinstance(result, ForwardRef): return result.evaluate(format=Format.FORWARDREF) @@ -781,14 +850,11 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluation): if not annotate.__closure__: - return None - freevars = annotate.__code__.co_freevars + return None, None new_closure = [] - for i, cell in enumerate(annotate.__closure__): - if i < len(freevars): - name = freevars[i] - else: - name = "__cell__" + cell_dict = {} + for name, cell in zip(annotate.__code__.co_freevars, annotate.__closure__, strict=True): + cell_dict[name] = cell new_cell = None if allow_evaluation: try: @@ -809,7 +875,7 @@ def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluat stringifier_dict.stringifiers.append(fwdref) new_cell = types.CellType(fwdref) new_closure.append(new_cell) - return tuple(new_closure) + return tuple(new_closure), cell_dict def _stringify_single(anno): @@ -851,7 +917,7 @@ def get_annotations( does not exist, the __annotate__ function is called. The FORWARDREF format uses __annotations__ if it exists and can be evaluated, and otherwise falls back to calling the __annotate__ function. - The SOURCE format tries __annotate__ first, and falls back to + The STRING format tries __annotate__ first, and falls back to using __annotations__, stringified using annotations_to_string(). This function handles several details for you: @@ -969,13 +1035,26 @@ def get_annotations( obj_globals = obj_locals = unwrap = None if unwrap is not None: + # Use an id-based visited set to detect cycles in the __wrapped__ + # and functools.partial.func chain (e.g. f.__wrapped__ = f). + # On cycle detection we stop and use whatever __globals__ we have + # found so far, mirroring the approach of inspect.unwrap(). + _seen_ids = {id(unwrap)} while True: if hasattr(unwrap, "__wrapped__"): - unwrap = unwrap.__wrapped__ + candidate = unwrap.__wrapped__ + if id(candidate) in _seen_ids: + break + _seen_ids.add(id(candidate)) + unwrap = candidate continue if functools := sys.modules.get("functools"): if isinstance(unwrap, functools.partial): - unwrap = unwrap.func + candidate = unwrap.func + if id(candidate) in _seen_ids: + break + _seen_ids.add(id(candidate)) + unwrap = candidate continue break if hasattr(unwrap, "__globals__"): @@ -995,7 +1074,8 @@ def get_annotations( locals = {param.__name__: param for param in type_params} | locals return_value = { - key: value if not isinstance(value, str) else eval(value, globals, locals) + key: value if not isinstance(value, str) + else eval(_rewrite_star_unpack(value), globals, locals) for key, value in ann.items() } return return_value @@ -1032,6 +1112,16 @@ def annotations_to_string(annotations): } +def _rewrite_star_unpack(arg): + """If the given argument annotation expression is a star unpack e.g. `'*Ts'` + rewrite it to a valid expression. + """ + if arg.lstrip().startswith("*"): + return f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] + else: + return arg + + def _get_and_call_annotate(obj, format): """Get the __annotate__ function and call it. @@ -1071,3 +1161,14 @@ def _get_dunder_annotations(obj): if not isinstance(ann, dict): raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") return ann + + +class _ExtraNameFixer(ast.NodeTransformer): + """Fixer for __extra_names__ items in ForwardRef __repr__ and string evaluation""" + def __init__(self, extra_names): + self.extra_names = extra_names + + def visit_Name(self, node: ast.Name): + if (new_name := self.extra_names.get(node.id, _sentinel)) is not _sentinel: + node = ast.Name(id=type_repr(new_name)) + return node diff --git a/Lib/argparse.py b/Lib/argparse.py index 2144c81886ad19f..29e6ebb9634261a 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -64,7 +64,6 @@ still considered an implementation detail.) """ -__version__ = '1.1' __all__ = [ 'ArgumentParser', 'ArgumentError', @@ -90,8 +89,10 @@ import os as _os import re as _re import sys as _sys +from gettext import gettext as _ +from gettext import ngettext -from gettext import gettext as _, ngettext +lazy import _colorize SUPPRESS = '==SUPPRESS==' @@ -149,10 +150,25 @@ def _copy_items(items): return copy.copy(items) +def _identity(value): + return value + + # =============== # Formatting Help # =============== +class _ColorlessTheme: + # A 'fake' theme for no colors + def __getattr__(self, name): + # _colorize's no_color themes are just all empty strings + # by directly using empty strings the import is avoided + if name.startswith("_"): + raise AttributeError(name) + return "" + +_colorless_theme = _ColorlessTheme() + class HelpFormatter(object): """Formatter for generating usage messages and argument help strings. @@ -167,7 +183,6 @@ def __init__( indent_increment=2, max_help_position=24, width=None, - color=True, ): # default setting for width if width is None: @@ -175,7 +190,6 @@ def __init__( width = shutil.get_terminal_size().columns width -= 2 - self._set_color(color) self._prog = prog self._indent_increment = indent_increment self._max_help_position = min(max_help_position, @@ -192,15 +206,35 @@ def __init__( self._whitespace_matcher = _re.compile(r'\s+', _re.ASCII) self._long_break_matcher = _re.compile(r'\n\n\n+') - def _set_color(self, color): - from _colorize import can_colorize, decolor, get_theme + self._set_color(False) + + def _set_color(self, color, *, file=None): + # Set a new color setting and file, clear caches for theme and decolor + self._theme_color = color + self._theme_file = file + self._cached_theme = None + self._cached_decolor = None - if color and can_colorize(): - self._theme = get_theme(force_color=True).argparse - self._decolor = decolor + def _get_theme_and_decolor(self): + # If self._theme_color is false, this prevents _colorize from importing + if self._theme_color and _colorize.can_colorize(file=self._theme_file): + self._cached_theme = _colorize.get_theme(force_color=True).argparse + self._cached_decolor = _colorize.decolor else: - self._theme = get_theme(force_no_color=True).argparse - self._decolor = lambda text: text + self._cached_theme = _colorless_theme + self._cached_decolor = _identity + + @property + def _theme(self): + if self._cached_theme is None: + self._get_theme_and_decolor() + return self._cached_theme + + @property + def _decolor(self): + if self._cached_decolor is None: + self._get_theme_and_decolor() + return self._cached_decolor # =============================== # Section and indentation methods @@ -281,7 +315,7 @@ def add_argument(self, action): if action.help is not SUPPRESS: # find all invocations - get_invocation = self._format_action_invocation + get_invocation = lambda x: self._decolor(self._format_action_invocation(x)) invocation_lengths = [len(get_invocation(action)) + self._current_indent] for subaction in self._iter_indented_subactions(action): invocation_lengths.append(len(get_invocation(subaction)) + self._current_indent) @@ -337,27 +371,17 @@ def _format_usage(self, usage, actions, groups, prefix): elif usage is None: prog = '%(prog)s' % dict(prog=self._prog) - # split optionals from positionals - optionals = [] - positionals = [] - for action in actions: - if action.option_strings: - optionals.append(action) - else: - positionals.append(action) - + parts, pos_start = self._get_actions_usage_parts(actions, groups) # build full usage string - format = self._format_actions_usage - action_usage = format(optionals + positionals, groups) - usage = ' '.join([s for s in [prog, action_usage] if s]) + usage = ' '.join(filter(None, [prog, *parts])) # wrap the usage parts if it's too long text_width = self._width - self._current_indent if len(prefix) + len(self._decolor(usage)) > text_width: # break usage into wrappable parts - opt_parts = self._get_actions_usage_parts(optionals, groups) - pos_parts = self._get_actions_usage_parts(positionals, groups) + opt_parts = parts[:pos_start] + pos_parts = parts[pos_start:] # helper for wrapping lines def get_lines(parts, indent, prefix=None): @@ -414,117 +438,141 @@ def get_lines(parts, indent, prefix=None): # prefix with 'usage:' return f'{t.usage}{prefix}{t.reset}{usage}\n\n' - def _format_actions_usage(self, actions, groups): - return ' '.join(self._get_actions_usage_parts(actions, groups)) - def _is_long_option(self, string): return len(string) > 2 def _get_actions_usage_parts(self, actions, groups): - # find group indices and identify actions in groups - group_actions = set() - inserts = {} - for group in groups: - if not group._group_actions: - raise ValueError(f'empty group {group}') + """Get usage parts with split index for optionals/positionals. - if all(action.help is SUPPRESS for action in group._group_actions): - continue - - try: - start = min(actions.index(item) for item in group._group_actions) - except ValueError: - continue - else: - end = start + len(group._group_actions) - if set(actions[start:end]) == set(group._group_actions): - group_actions.update(group._group_actions) - inserts[start, end] = group + Returns (parts, pos_start) where pos_start is the index in parts + where positionals begin. + This preserves mutually exclusive group formatting across the + optionals/positionals boundary (gh-75949). + """ + actions = [action for action in actions if action.help is not SUPPRESS] + # group actions by mutually exclusive groups + action_groups = dict.fromkeys(actions) + for group in groups: + for action in group._group_actions: + if action in action_groups: + action_groups[action] = group + # positional arguments keep their position + positionals = [] + for action in actions: + if not action.option_strings: + group = action_groups.pop(action) + if group: + group_actions = [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + [action] + positionals.append((group.required, group_actions)) + else: + positionals.append((None, [action])) + # the remaining optional arguments are sorted by the position of + # the first option in the group + optionals = [] + for action in actions: + if action.option_strings and action in action_groups: + group = action_groups.pop(action) + if group: + group_actions = [action] + [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + optionals.append((group.required, group_actions)) + else: + optionals.append((None, [action])) # collect all actions format strings parts = [] t = self._theme - for action in actions: - - # suppressed arguments are marked with None - if action.help is SUPPRESS: - part = None - - # produce all arg strings - elif not action.option_strings: - default = self._get_default_metavar_for_positional(action) - part = ( - t.summary_action - + self._format_args(action, default) - + t.reset - ) - - # if it's in a group, strip the outer [] - if action in group_actions: - if part[0] == '[' and part[-1] == ']': - part = part[1:-1] - - # produce the first way to invoke the option in brackets - else: - option_string = action.option_strings[0] - if self._is_long_option(option_string): - option_color = t.summary_long_option + pos_start = None + for i, (required, group) in enumerate(optionals + positionals): + start = len(parts) + if i == len(optionals): + pos_start = start + in_group = len(group) > 1 + for action in group: + # produce all arg strings + if not action.option_strings: + default = self._get_default_metavar_for_positional(action) + part = self._format_args(action, default) + # if it's in a group, strip the outer [] + if in_group: + if part[0] == '[' and part[-1] == ']': + part = part[1:-1] + part = t.summary_action + part + t.reset + + # produce the first way to invoke the option in brackets else: - option_color = t.summary_short_option + option_string = action.option_strings[0] + if self._is_long_option(option_string): + option_color = t.summary_long_option + else: + option_color = t.summary_short_option - # if the Optional doesn't take a value, format is: - # -s or --long - if action.nargs == 0: - part = action.format_usage() - part = f"{option_color}{part}{t.reset}" + # if the Optional doesn't take a value, format is: + # -s or --long + if action.nargs == 0: + part = action.format_usage() + part = f"{option_color}{part}{t.reset}" - # if the Optional takes a value, format is: - # -s ARGS or --long ARGS - else: - default = self._get_default_metavar_for_optional(action) - args_string = self._format_args(action, default) - part = ( - f"{option_color}{option_string} " - f"{t.summary_label}{args_string}{t.reset}" - ) - - # make it look optional if it's not required or in a group - if not action.required and action not in group_actions: - part = '[%s]' % part - - # add the action string to the list - parts.append(part) - - # group mutually exclusive actions - inserted_separators_indices = set() - for start, end in sorted(inserts, reverse=True): - group = inserts[start, end] - group_parts = [item for item in parts[start:end] if item is not None] - group_size = len(group_parts) - if group.required: - open, close = "()" if group_size > 1 else ("", "") - else: - open, close = "[]" - group_parts[0] = open + group_parts[0] - group_parts[-1] = group_parts[-1] + close - for i, part in enumerate(group_parts[:-1], start=start): - # insert a separator if not already done in a nested group - if i not in inserted_separators_indices: - parts[i] = part + ' |' - inserted_separators_indices.add(i) - parts[start + group_size - 1] = group_parts[-1] - for i in range(start + group_size, end): - parts[i] = None - - # return the usage parts - return [item for item in parts if item is not None] + # if the Optional takes a value, format is: + # -s ARGS or --long ARGS + else: + default = self._get_default_metavar_for_optional(action) + args_string = self._format_args(action, default) + part = ( + f"{option_color}{option_string} " + f"{t.summary_label}{args_string}{t.reset}" + ) + + # make it look optional if it's not required or in a group + if not (action.required or required or in_group): + part = '[%s]' % part + + # add the action string to the list + parts.append(part) + + if in_group: + parts[start] = ('(' if required else '[') + parts[start] + for i in range(start, len(parts) - 1): + parts[i] += ' |' + parts[-1] += ')' if required else ']' + + if pos_start is None: + pos_start = len(parts) + return parts, pos_start def _format_text(self, text): if '%(prog)' in text: text = text % dict(prog=self._prog) text_width = max(self._width - self._current_indent, 11) indent = ' ' * self._current_indent - return self._fill_text(text, text_width, indent) + '\n\n' + text = self._fill_text(text, text_width, indent) + text = self._apply_text_markup(text) + return text + '\n\n' + + def _apply_text_markup(self, text): + """Apply color markup to text. + + Supported markup: + `...` or ``...`` - inline code (rendered with prog_extra color) + + When colors are disabled, backticks are preserved as-is. + """ + t = self._theme + if not t.reset: + return text + text = _re.sub( + r'(`{1,2})([^`]+)\1', + rf'{t.prog_extra}\2{t.reset}', + text, + ) + return text def _format_action(self, action): # determine the required width and the entry label @@ -665,7 +713,7 @@ def _format_args(self, action, default_metavar): def _expand_help(self, action): help_string = self._get_help_string(action) if '%' not in help_string: - return help_string + return self._apply_text_markup(help_string) params = dict(vars(action), prog=self._prog) for name in list(params): value = params[name] @@ -675,7 +723,43 @@ def _expand_help(self, action): params[name] = value.__name__ if params.get('choices') is not None: params['choices'] = ', '.join(map(str, params['choices'])) - return help_string % params + + t = self._theme + + result = help_string % params + + if not t.reset: + return result + + # Match format specifiers like: %s, %d, %(key)s, etc. + fmt_spec = r''' + % + (?: + % # %% escape + | + (?:\((?P[^)]*)\))? # key + [-#0\ +]* # flags + (?:\*|\d+)? # width + (?:\.(?:\*|\d+))? # precision + [hlL]? # length modifier + [diouxXeEfFgGcrsa] # conversion type + ) + ''' + + def colorize(match): + spec, key = match.group(0, 'key') + if spec == '%%': + return '%' + if key is not None: + # %(key)... - format and colorize + formatted = spec % {key: params[key]} + return f'{t.interpolated_value}{formatted}{t.reset}' + # bare %s etc. - format with full params dict, no colorization + return spec % params + + return self._apply_text_markup( + _re.sub(fmt_spec, colorize, help_string, flags=_re.VERBOSE) + ) def _iter_indented_subactions(self, action): try: @@ -745,11 +829,21 @@ def _get_help_string(self, action): if help is None: help = '' - if '%(default)' not in help: - if action.default is not SUPPRESS: - defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] - if action.option_strings or action.nargs in defaulting_nargs: - help += _(' (default: %(default)s)') + if ( + '%(default)' not in help + and action.default is not SUPPRESS + and not action.required + ): + defaulting_nargs = (OPTIONAL, ZERO_OR_MORE) + if action.option_strings or action.nargs in defaulting_nargs: + t = self._theme + default_str = _(" (default: %(default)s)") + prefix, suffix = default_str.split("%(default)s") + help += ( + f" {t.default}{prefix.lstrip()}{t.reset}" + f"%(default)s" + f"{t.default}{suffix}{t.reset}" + ) return help @@ -933,15 +1027,26 @@ def __init__(self, deprecated=False): _option_strings = [] + neg_option_strings = [] for option_string in option_strings: _option_strings.append(option_string) - if option_string.startswith('--'): - if option_string.startswith('--no-'): + if len(option_string) > 2 and option_string[0] == option_string[1]: + # two-dash long option: '--foo' -> '--no-foo' + if option_string.startswith('no-', 2): + raise ValueError(f'invalid option name {option_string!r} ' + f'for BooleanOptionalAction') + option_string = option_string[:2] + 'no-' + option_string[2:] + _option_strings.append(option_string) + neg_option_strings.append(option_string) + elif len(option_string) > 2 and option_string[0] != option_string[1]: + # single-dash long option: '-foo' -> '-nofoo' + if option_string.startswith('no', 1): raise ValueError(f'invalid option name {option_string!r} ' f'for BooleanOptionalAction') - option_string = '--no-' + option_string[2:] + option_string = option_string[:1] + 'no' + option_string[1:] _option_strings.append(option_string) + neg_option_strings.append(option_string) super().__init__( option_strings=_option_strings, @@ -951,11 +1056,12 @@ def __init__(self, required=required, help=help, deprecated=deprecated) + self.neg_option_strings = neg_option_strings def __call__(self, parser, namespace, values, option_string=None): if option_string in self.option_strings: - setattr(namespace, self.dest, not option_string.startswith('--no-')) + setattr(namespace, self.dest, option_string not in self.neg_option_strings) def format_usage(self): return ' | '.join(self.option_strings) @@ -1552,8 +1658,8 @@ def add_argument(self, *args, **kwargs): f'instance of it must be passed') # raise an error if the metavar does not match the type - if hasattr(self, "_get_formatter"): - formatter = self._get_formatter() + if hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: formatter._format_args(action, None) except TypeError: @@ -1661,29 +1767,35 @@ def _get_positional_kwargs(self, dest, **kwargs): def _get_optional_kwargs(self, *args, **kwargs): # determine short and long option strings option_strings = [] - long_option_strings = [] for option_string in args: # error on strings that don't start with an appropriate prefix - if not option_string[0] in self.prefix_chars: + if option_string[0] not in self.prefix_chars: raise ValueError( f'invalid option string {option_string!r}: ' f'must start with a character {self.prefix_chars!r}') - - # strings starting with two prefix characters are long options option_strings.append(option_string) - if len(option_string) > 1 and option_string[1] in self.prefix_chars: - long_option_strings.append(option_string) # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x' dest = kwargs.pop('dest', None) if dest is None: - if long_option_strings: - dest_option_string = long_option_strings[0] - else: - dest_option_string = option_strings[0] - dest = dest_option_string.lstrip(self.prefix_chars) + priority = 0 + for option_string in option_strings: + if len(option_string) <= 2: + # short option: '-x' -> 'x' + if priority < 1: + dest = option_string.lstrip(self.prefix_chars) + priority = 1 + elif option_string[1] not in self.prefix_chars: + # single-dash long option: '-foo' -> 'foo' + if priority < 2: + dest = option_string.lstrip(self.prefix_chars) + priority = 2 + else: + # two-dash long option: '--foo' -> 'foo' + dest = option_string.lstrip(self.prefix_chars) + break if not dest: - msg = f'dest= is required for options like {option_string!r}' + msg = f'dest= is required for options like {repr(option_strings)[1:-1]}' raise TypeError(msg) dest = dest.replace('-', '_') @@ -1741,8 +1853,8 @@ def _handle_conflict_resolve(self, action, conflicting_actions): action.container._remove_action(action) def _check_help(self, action): - if action.help and hasattr(self, "_get_formatter"): - formatter = self._get_formatter() + if action.help and hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: formatter._expand_help(action) except (ValueError, TypeError, KeyError) as exc: @@ -1858,7 +1970,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer): - exit_on_error -- Determines whether or not ArgumentParser exits with error info when an error occurs - suggest_on_error - Enables suggestions for mistyped argument choices - and subparser names (default: ``False``) + and subparser names (default: ``True``) - color - Allow color output in help messages (default: ``False``) """ @@ -1877,7 +1989,7 @@ def __init__(self, allow_abbrev=True, exit_on_error=True, *, - suggest_on_error=False, + suggest_on_error=True, color=True, ): superinit = super(ArgumentParser, self).__init__ @@ -1897,15 +2009,16 @@ def __init__(self, self.suggest_on_error = suggest_on_error self.color = color + # Cached formatter for validation (avoids repeated _set_color calls) + self._cached_formatter = None + add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) self._optionals = add_group(_('options')) self._subparsers = None # register types - def identity(string): - return string - self.register('type', None, identity) + self.register('type', None, _identity) # add help argument if necessary # (using explicit default to override global argument_default) @@ -1958,12 +2071,16 @@ def add_subparsers(self, **kwargs): self._subparsers = self._positionals # prog defaults to the usage message of this parser, skipping - # optional arguments and with no "usage:" prefix + # non-required optional arguments and with no "usage:" prefix if kwargs.get('prog') is None: - formatter = self._get_formatter() + # Create formatter without color to avoid storing ANSI codes in prog + formatter = self.formatter_class(prog=self.prog) + formatter._set_color(False) positionals = self._get_positional_actions() + required_optionals = [action for action in self._get_optional_actions() + if action.required] groups = self._mutually_exclusive_groups - formatter.add_usage(None, positionals, groups, '') + formatter.add_usage(None, required_optionals + positionals, groups, '') kwargs['prog'] = formatter.format_help().strip() # create the parsers action and add it to the positionals list @@ -2430,7 +2547,7 @@ def _parse_optional(self, arg_string): return None # if it doesn't start with a prefix, it was meant to be positional - if not arg_string[0] in self.prefix_chars: + if arg_string[0] not in self.prefix_chars: return None # if the option string is present in the parser, return the action @@ -2539,7 +2656,7 @@ def _get_nargs_pattern(self, action): # allow any number of options or arguments elif nargs == REMAINDER: - nargs_pattern = '([AO]*)' if option else '(.*)' + nargs_pattern = '(.*)' # allow one argument followed by any number of options or arguments elif nargs == PARSER: @@ -2674,7 +2791,7 @@ def _check_value(self, action, value): if value not in choices: args = {'value': str(value), - 'choices': ', '.join(map(str, action.choices))} + 'choices': ', '.join(repr(str(choice)) for choice in action.choices)} msg = _('invalid choice: %(value)r (choose from %(choices)s)') if self.suggest_on_error and isinstance(value, str): @@ -2692,14 +2809,16 @@ def _check_value(self, action, value): # Help-formatting methods # ======================= - def format_usage(self): - formatter = self._get_formatter() + def format_usage(self, formatter=None): + if formatter is None: + formatter = self._get_formatter() formatter.add_usage(self.usage, self._actions, self._mutually_exclusive_groups) return formatter.format_help() - def format_help(self): - formatter = self._get_formatter() + def format_help(self, formatter=None): + if formatter is None: + formatter = self._get_formatter() # usage formatter.add_usage(self.usage, self._actions, @@ -2721,11 +2840,22 @@ def format_help(self): # determine help from format above return formatter.format_help() - def _get_formatter(self): + def _get_formatter(self, file=None): formatter = self.formatter_class(prog=self.prog) - formatter._set_color(self.color) + formatter._set_color(self.color, file=file) return formatter + def _get_validation_formatter(self): + # Return cached formatter for read-only validation operations + # (_expand_help and _format_args). Avoids repeated slow _set_color calls. + # Validation never renders output, so force color off to avoid + # importing _colorize during add_argument. + if self._cached_formatter is None: + formatter = self.formatter_class(prog=self.prog) + formatter._set_color(False) + self._cached_formatter = formatter + return self._cached_formatter + # ===================== # Help-printing methods # ===================== @@ -2733,12 +2863,26 @@ def _get_formatter(self): def print_usage(self, file=None): if file is None: file = _sys.stdout - self._print_message(self.format_usage(), file) + formatter = self._get_formatter(file=file) + try: + usage_text = self.format_usage(formatter=formatter) + except TypeError: + # Backward compatibility for formatter classes that + # do not accept the 'formatter' keyword argument. + usage_text = self.format_usage() + self._print_message(usage_text, file) def print_help(self, file=None): if file is None: file = _sys.stdout - self._print_message(self.format_help(), file) + formatter = self._get_formatter(file=file) + try: + help_text = self.format_help(formatter=formatter) + except TypeError: + # Backward compatibility for formatter classes that + # do not accept the 'formatter' keyword argument. + help_text = self.format_help() + self._print_message(help_text, file) def _print_message(self, message, file=None): if message: @@ -2748,6 +2892,13 @@ def _print_message(self, message, file=None): except (AttributeError, OSError): pass + def _get_theme(self, file=None): + # If self.color is False, _colorize is not imported + if self.color and _colorize.can_colorize(file=file): + return _colorize.get_theme(force_color=True).argparse + else: + return _colorless_theme + # =============== # Exiting methods # =============== @@ -2767,9 +2918,26 @@ def error(self, message): should either exit or raise an exception. """ self.print_usage(_sys.stderr) + theme = self._get_theme(file=_sys.stderr) + fmt = _('%(prog)s: error: %(message)s\n') + fmt = fmt.replace('error: %(message)s', + f'{theme.error}error:{theme.reset} {theme.message}%(message)s{theme.reset}') + args = {'prog': self.prog, 'message': message} - self.exit(2, _('%(prog)s: error: %(message)s\n') % args) + self.exit(2, fmt % args) def _warning(self, message): + theme = self._get_theme(file=_sys.stderr) + fmt = _('%(prog)s: warning: %(message)s\n') + fmt = fmt.replace('warning: %(message)s', + f'{theme.warning}warning:{theme.reset} {theme.message}%(message)s{theme.reset}') args = {'prog': self.prog, 'message': message} - self._print_message(_('%(prog)s: warning: %(message)s\n') % args, _sys.stderr) + self._print_message(fmt % args, _sys.stderr) + +def __getattr__(name): + if name == "__version__": + from warnings import _deprecated + + _deprecated("__version__", remove=(3, 20)) + return "1.1" # Do not change + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/ast.py b/Lib/ast.py index 6d3daf64f5c6d75..f445b32040e5fb2 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -21,10 +21,10 @@ :license: Python License. """ from _ast import * - +lazy import warnings def parse(source, filename='', mode='exec', *, - type_comments=False, feature_version=None, optimize=-1): + type_comments=False, feature_version=None, optimize=-1, module=None): """ Parse the source into an AST node. Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). @@ -44,7 +44,8 @@ def parse(source, filename='', mode='exec', *, feature_version = minor # Else it should be an int giving the minor version for 3.x. return compile(source, filename, mode, flags, - _feature_version=feature_version, optimize=optimize) + _feature_version=feature_version, optimize=optimize, + module=module) def literal_eval(node_or_string): @@ -57,73 +58,93 @@ def literal_eval(node_or_string): Caution: A complex expression can overflow the C stack and cause a crash. """ if isinstance(node_or_string, str): - node_or_string = parse(node_or_string.lstrip(" \t"), mode='eval') - if isinstance(node_or_string, Expression): + node_or_string = parse(node_or_string.lstrip(" \t"), mode='eval').body + elif isinstance(node_or_string, Expression): node_or_string = node_or_string.body - def _raise_malformed_node(node): - msg = "malformed node or string" - if lno := getattr(node, 'lineno', None): - msg += f' on line {lno}' - raise ValueError(msg + f': {node!r}') - def _convert_num(node): - if not isinstance(node, Constant) or type(node.value) not in (int, float, complex): - _raise_malformed_node(node) + return _convert_literal(node_or_string) + + +def _convert_literal(node): + """ + Used by `literal_eval` to convert an AST node into a value. + """ + if isinstance(node, Constant): return node.value - def _convert_signed_num(node): - if isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)): - operand = _convert_num(node.operand) - if isinstance(node.op, UAdd): - return + operand - else: - return - operand - return _convert_num(node) - def _convert(node): - if isinstance(node, Constant): - return node.value - elif isinstance(node, Tuple): - return tuple(map(_convert, node.elts)) - elif isinstance(node, List): - return list(map(_convert, node.elts)) - elif isinstance(node, Set): - return set(map(_convert, node.elts)) - elif (isinstance(node, Call) and isinstance(node.func, Name) and - node.func.id == 'set' and node.args == node.keywords == []): - return set() - elif isinstance(node, Dict): - if len(node.keys) != len(node.values): - _raise_malformed_node(node) - return dict(zip(map(_convert, node.keys), - map(_convert, node.values))) - elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)): - left = _convert_signed_num(node.left) - right = _convert_num(node.right) - if isinstance(left, (int, float)) and isinstance(right, complex): - if isinstance(node.op, Add): - return left + right - else: - return left - right - return _convert_signed_num(node) - return _convert(node_or_string) + if isinstance(node, Dict) and len(node.keys) == len(node.values): + return dict(zip( + map(_convert_literal, node.keys), + map(_convert_literal, node.values), + )) + if isinstance(node, Tuple): + return tuple(map(_convert_literal, node.elts)) + if isinstance(node, List): + return list(map(_convert_literal, node.elts)) + if isinstance(node, Set): + return set(map(_convert_literal, node.elts)) + if ( + isinstance(node, Call) and isinstance(node.func, Name) + and node.func.id == 'set' and node.args == node.keywords == [] + ): + return set() + if ( + isinstance(node, UnaryOp) + and isinstance(node.op, (UAdd, USub)) + and isinstance(node.operand, Constant) + and type(operand := node.operand.value) in (int, float, complex) + ): + if isinstance(node.op, UAdd): + return + operand + else: + return - operand + if ( + isinstance(node, BinOp) + and isinstance(node.op, (Add, Sub)) + and isinstance(node.left, (Constant, UnaryOp)) + and isinstance(node.right, Constant) + and type(left := _convert_literal(node.left)) in (int, float) + and type(right := _convert_literal(node.right)) is complex + ): + if isinstance(node.op, Add): + return left + right + else: + return left - right + msg = "malformed node or string" + if lno := getattr(node, 'lineno', None): + msg += f' on line {lno}' + raise ValueError(msg + f': {node!r}') def dump( node, annotate_fields=True, include_attributes=False, *, - indent=None, show_empty=False, + color=False, indent=None, show_empty=False, ): """ Return a formatted dump of the tree in node. This is mainly useful for - debugging purposes. If annotate_fields is true (by default), - the returned string will show the names and the values for fields. - If annotate_fields is false, the result string will be more compact by - omitting unambiguous field names. Attributes such as line - numbers and column offsets are not dumped by default. If this is wanted, - include_attributes can be set to true. If indent is a non-negative - integer or string, then the tree will be pretty-printed with that indent - level. None (the default) selects the single line representation. + debugging purposes. + + If annotate_fields is true (by default), the returned string will show the + names and the values for fields. If annotate_fields is false, the result + string will be more compact by omitting unambiguous field names. + + Attributes such as line numbers and column offsets are not dumped by default. + If this is wanted, include_attributes can be set to true. + + If color is true, the returned string is syntax highlighted using ANSI + escape sequences. If color is false (the default), colored output is always + disabled. + + If indent is a non-negative integer or string, then the tree will be + pretty-printed with that indent level. If indent is None (the default), + the tree is dumped on a single line. + If show_empty is False, then empty lists and fields that are None will be omitted from the output for better readability. """ + from _colorize import get_theme + + t = get_theme(force_color=color, force_no_color=not color).ast + def _format(node, level=0): if indent is not None: level += 1 @@ -158,7 +179,9 @@ def _format(node, level=0): field_type = cls._field_types.get(name, object) if field_type is expr_context: if not keywords: - args_buffer.append(repr(value)) + args_buffer.append( + f'{t.node}{type(value).__name__}' + f'{t.reset}()') continue if not keywords: args.extend(args_buffer) @@ -166,7 +189,7 @@ def _format(node, level=0): value, simple = _format(value, level) allsimple = allsimple and simple if keywords: - args.append('%s=%s' % (name, value)) + args.append(f'{t.field}{name}{t.reset}={value}') else: args.append(value) if include_attributes and node._attributes: @@ -179,14 +202,21 @@ def _format(node, level=0): continue value, simple = _format(value, level) allsimple = allsimple and simple - args.append('%s=%s' % (name, value)) + args.append(f'{t.attribute}{name}{t.reset}={value}') + cls_name = f'{t.node}{cls.__name__}{t.reset}' if allsimple and len(args) <= 3: - return '%s(%s)' % (node.__class__.__name__, ', '.join(args)), not args - return '%s(%s%s)' % (node.__class__.__name__, prefix, sep.join(args)), False + return f'{cls_name}({", ".join(args)})', not args + return f'{cls_name}({prefix}{sep.join(args)})', False elif isinstance(node, list): if not node: return '[]', True return '[%s%s]' % (prefix, sep.join(_format(x, level)[0] for x in node)), False + if isinstance(node, bool) or node is None or node is Ellipsis: + return f'{t.keyword}{node!r}{t.reset}', True + if isinstance(node, (int, float, complex)): + return f'{t.number}{node!r}{t.reset}', True + if isinstance(node, (str, bytes)): + return f'{t.string}{node!r}{t.reset}', True return repr(node), True if not isinstance(node, AST): @@ -600,9 +630,11 @@ def __new__(cls, dims=(), **kwargs): def _dims_getter(self): """Deprecated. Use elts instead.""" + warnings._deprecated(f"ast.Tuple.dims", remove=(3, 21)) return self.elts def _dims_setter(self, value): + warnings._deprecated(f"ast.Tuple.dims", remove=(3, 21)) self.elts = value Tuple.dims = property(_dims_getter, _dims_setter) @@ -634,9 +666,9 @@ def main(args=None): import argparse import sys - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('infile', nargs='?', default='-', - help='the file to parse; defaults to stdin') + help='the file to parse; defaults to `stdin`') parser.add_argument('-m', '--mode', default='exec', choices=('exec', 'single', 'eval', 'func_type'), help='specify what kind of code must be parsed') @@ -649,11 +681,11 @@ def main(args=None): help='indentation of nodes (number of spaces)') parser.add_argument('--feature-version', type=str, default=None, metavar='VERSION', - help='Python version in the format 3.x ' - '(for example, 3.10)') + help='Python version in the format `3.x` ' + '(for example, `3.10`)') parser.add_argument('-O', '--optimize', type=int, default=-1, metavar='LEVEL', - help='optimization level for parser (default -1)') + help='optimization level for parser') parser.add_argument('--show-empty', default=False, action='store_true', help='show empty lists and fields in dump output') args = parser.parse_args(args) @@ -679,8 +711,28 @@ def main(args=None): tree = parse(source, name, args.mode, type_comments=args.no_type_comments, feature_version=feature_version, optimize=args.optimize) + from _colorize import can_colorize print(dump(tree, include_attributes=args.include_attributes, + color=can_colorize(file=sys.stdout), indent=args.indent, show_empty=args.show_empty)) +_deprecated = { + 'slice': globals().pop("slice"), + 'Index': globals().pop("Index"), + 'ExtSlice': globals().pop("ExtSlice"), + 'Suite': globals().pop("Suite"), + 'AugLoad': globals().pop("AugLoad"), + 'AugStore': globals().pop("AugStore"), + 'Param': globals().pop("Param") +} + +def __getattr__(attr): + try: + val = _deprecated[attr] + except KeyError: + raise AttributeError(f"module 'ast' has no attribute {attr!r}") from None + warnings._deprecated(f"ast.{attr}", remove=(3, 21)) + return val + if __name__ == '__main__': main() diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index ff3a69d1e172971..7f0565d0b8ddc76 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -12,13 +12,16 @@ import types import warnings -from _colorize import get_theme -from _pyrepl.console import InteractiveColoredConsole +try: + from _colorize import get_theme + from _pyrepl.console import InteractiveColoredConsole as InteractiveConsole +except ModuleNotFoundError: + from code import InteractiveConsole from . import futures -class AsyncIOInteractiveConsole(InteractiveColoredConsole): +class AsyncIOInteractiveConsole(InteractiveConsole): def __init__(self, locals, loop): super().__init__(locals, filename="") @@ -74,7 +77,8 @@ def callback(): return except BaseException: if keyboard_interrupted: - self.write("\nKeyboardInterrupt\n") + if not CAN_USE_PYREPL: + self.write("\nKeyboardInterrupt\n") else: self.showtraceback() return self.STATEMENT_FAILED @@ -85,34 +89,43 @@ def run(self): global return_code try: - banner = ( - f'asyncio REPL {sys.version} on {sys.platform}\n' - f'Use "await" directly instead of "asyncio.run()".\n' - f'Type "help", "copyright", "credits" or "license" ' - f'for more information.\n' - ) + if not sys.flags.quiet: + banner = ( + f'asyncio REPL {sys.version} on {sys.platform}\n' + f'Use "await" directly instead of "asyncio.run()".\n' + f'Type "help", "copyright", "credits" or "license" ' + f'for more information.\n' + ) - console.write(banner) + console.write(banner) - if startup_path := os.getenv("PYTHONSTARTUP"): + if not sys.flags.isolated and (startup_path := os.getenv("PYTHONSTARTUP")): sys.audit("cpython.run_startup", startup_path) - - import tokenize - with tokenize.open(startup_path) as f: - startup_code = compile(f.read(), startup_path, "exec") + try: + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") exec(startup_code, console.locals) + except SystemExit: + raise + except BaseException: + console.showtraceback() ps1 = getattr(sys, "ps1", ">>> ") if CAN_USE_PYREPL: theme = get_theme().syntax ps1 = f"{theme.prompt}{ps1}{theme.reset}" - console.write(f"{ps1}import asyncio\n") + import_line = f'{theme.keyword}import{theme.reset} asyncio' + else: + import_line = "import asyncio" + console.write(f"{ps1}{import_line}\n") if CAN_USE_PYREPL: from _pyrepl.simple_interact import ( run_multiline_interactive_console, ) try: + sys.ps1 = ps1 run_multiline_interactive_console(console) except SystemExit: # expected via the `exit` and `quit` commands @@ -146,24 +159,35 @@ def interrupt(self) -> None: parser = argparse.ArgumentParser( prog="python3 -m asyncio", description="Interactive asyncio shell and CLI tools", - color=True, ) subparsers = parser.add_subparsers(help="sub-commands", dest="command") ps = subparsers.add_parser( "ps", help="Display a table of all pending tasks in a process" ) ps.add_argument("pid", type=int, help="Process ID to inspect") + ps.add_argument( + "--retries", + type=int, + default=3, + help="Number of retries on transient attach errors", + ) pstree = subparsers.add_parser( "pstree", help="Display a tree of all pending tasks in a process" ) pstree.add_argument("pid", type=int, help="Process ID to inspect") + pstree.add_argument( + "--retries", + type=int, + default=3, + help="Number of retries on transient attach errors", + ) args = parser.parse_args() match args.command: case "ps": - asyncio.tools.display_awaited_by_tasks_table(args.pid) + asyncio.tools.display_awaited_by_tasks_table(args.pid, retries=args.retries) sys.exit(0) case "pstree": - asyncio.tools.display_awaited_by_tasks_tree(args.pid) + asyncio.tools.display_awaited_by_tasks_tree(args.pid, retries=args.retries) sys.exit(0) case None: pass # continue to the interactive shell @@ -179,7 +203,10 @@ def interrupt(self) -> None: if os.getenv('PYTHON_BASIC_REPL'): CAN_USE_PYREPL = False else: - from _pyrepl.main import CAN_USE_PYREPL + try: + from _pyrepl.main import CAN_USE_PYREPL + except ModuleNotFoundError: + CAN_USE_PYREPL = False return_code = 0 loop = asyncio.new_event_loop() @@ -235,4 +262,5 @@ def interrupt(self) -> None: break console.write('exiting asyncio REPL...\n') + loop.close() sys.exit(return_code) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 520d4b398545bf1..e6c72e3d5b5487e 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -14,21 +14,24 @@ """ import collections +import contextvars import collections.abc import concurrent.futures import errno import heapq import itertools +import math import os import socket import stat import subprocess +import sys import threading import time import traceback -import sys import warnings import weakref +import inspect try: import ssl @@ -289,6 +292,7 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, self._ssl_shutdown_timeout = ssl_shutdown_timeout self._serving = False self._serving_forever_fut = None + self._context = contextvars.copy_context() def __repr__(self): return f'<{self.__class__.__name__} sockets={self.sockets!r}>' @@ -318,7 +322,7 @@ def _start_serving(self): self._loop._start_serving( self._protocol_factory, sock, self._ssl_context, self, self._backlog, self._ssl_handshake_timeout, - self._ssl_shutdown_timeout) + self._ssl_shutdown_timeout, context=self._context) def get_loop(self): return self._loop @@ -380,6 +384,7 @@ async def serve_forever(self): except exceptions.CancelledError: try: self.close() + self.close_clients() await self.wait_closed() finally: raise @@ -483,10 +488,10 @@ def set_task_factory(self, factory): If factory is None the default task factory will be set. If factory is a callable, it should have a signature matching - '(loop, coro, **kwargs)', where 'loop' will be a reference to the active - event loop, 'coro' will be a coroutine object, and **kwargs will be - arbitrary keyword arguments that should be passed on to Task. - The callable must return a Task. + '(loop, coro, **kwargs)', where 'loop' will be a reference to the + active event loop, 'coro' will be a coroutine object, and **kwargs + will be arbitrary keyword arguments that should be passed on to + Task. The callable must return a Task. """ if factory is not None and not callable(factory): raise TypeError('task factory must be a callable or None') @@ -507,7 +512,8 @@ def _make_ssl_transport( extra=None, server=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, - call_connection_made=True): + call_connection_made=True, + context=None): """Create SSL transport.""" raise NotImplementedError @@ -636,7 +642,7 @@ def _check_running(self): def _run_forever_setup(self): """Prepare the run loop to process events. - This method exists so that custom custom event loop subclasses (e.g., event loops + This method exists so that custom event loop subclasses (e.g., event loops that integrate a GUI event loop with Python's event loop) have access to all the loop setup logic. """ @@ -656,7 +662,7 @@ def _run_forever_setup(self): def _run_forever_cleanup(self): """Clean up after an event loop finishes the looping over events. - This method exists so that custom custom event loop subclasses (e.g., event loops + This method exists so that custom event loop subclasses (e.g., event loops that integrate a GUI event loop with Python's event loop) have access to all the loop cleanup logic. """ @@ -721,8 +727,8 @@ def run_until_complete(self, future): def stop(self): """Stop running the event loop. - Every callback already scheduled will still run. This simply informs - run_forever to stop looping after a complete iteration. + Every callback already scheduled will still run. This simply + informs run_forever to stop looping after a complete iteration. """ self._stopping = True @@ -835,7 +841,7 @@ def call_soon(self, callback, *args, context=None): def _check_callback(self, callback, method): if (coroutines.iscoroutine(callback) or - coroutines._iscoroutinefunction(callback)): + inspect.iscoroutinefunction(callback)): raise TypeError( f"coroutines cannot be used with {method}()") if not callable(callback): @@ -949,7 +955,7 @@ async def sock_sendfile(self, sock, file, offset=0, count=None, try: return await self._sock_sendfile_native(sock, file, offset, count) - except exceptions.SendfileNotAvailableError as exc: + except exceptions.SendfileNotAvailableError: if not fallback: raise return await self._sock_sendfile_fallback(sock, file, @@ -963,7 +969,7 @@ async def _sock_sendfile_native(self, sock, file, offset, count): f"and file {file!r} combination") async def _sock_sendfile_fallback(self, sock, file, offset, count): - if offset: + if hasattr(file, 'seek'): file.seek(offset) blocksize = ( min(count, constants.SENDFILE_FALLBACK_READBUFFER_SIZE) @@ -1070,12 +1076,12 @@ async def create_connection( Create a streaming transport connection to a given internet host and port: socket family AF_INET or socket.AF_INET6 depending on host (or - family if specified), socket type SOCK_STREAM. protocol_factory must be - a callable returning a protocol instance. + family if specified), socket type SOCK_STREAM. protocol_factory must + be a callable returning a protocol instance. - This method is a coroutine which will try to establish the connection - in the background. When successful, the coroutine returns a - (transport, protocol) pair. + This method is a coroutine which will try to establish the + connection in the background. When successful, the coroutine + returns a (transport, protocol) pair. """ if server_hostname is not None and not ssl: raise ValueError('server_hostname is only meaningful with ssl') @@ -1211,9 +1217,10 @@ async def _create_connection_transport( self, sock, protocol_factory, ssl, server_hostname, server_side=False, ssl_handshake_timeout=None, - ssl_shutdown_timeout=None): + ssl_shutdown_timeout=None, context=None): sock.setblocking(False) + context = context if context is not None else contextvars.copy_context() protocol = protocol_factory() waiter = self.create_future() @@ -1223,9 +1230,10 @@ async def _create_connection_transport( sock, protocol, sslcontext, waiter, server_side=server_side, server_hostname=server_hostname, ssl_handshake_timeout=ssl_handshake_timeout, - ssl_shutdown_timeout=ssl_shutdown_timeout) + ssl_shutdown_timeout=ssl_shutdown_timeout, + context=context) else: - transport = self._make_socket_transport(sock, protocol, waiter) + transport = self._make_socket_transport(sock, protocol, waiter, context=context) try: await waiter @@ -1270,7 +1278,7 @@ async def sendfile(self, transport, file, offset=0, count=None, try: return await self._sendfile_native(transport, file, offset, count) - except exceptions.SendfileNotAvailableError as exc: + except exceptions.SendfileNotAvailableError: if not fallback: raise @@ -1278,7 +1286,6 @@ async def sendfile(self, transport, file, offset=0, count=None, raise RuntimeError( f"fallback is disabled and native sendfile is not " f"supported for transport {transport!r}") - return await self._sendfile_fallback(transport, file, offset, count) @@ -1287,7 +1294,7 @@ async def _sendfile_native(self, transp, file, offset, count): "sendfile syscall is not supported") async def _sendfile_fallback(self, transp, file, offset, count): - if offset: + if hasattr(file, 'seek'): file.seek(offset) blocksize = min(count, 16384) if count else 16384 buf = bytearray(blocksize) @@ -1345,6 +1352,17 @@ async def start_tls(self, transport, protocol, sslcontext, *, # have a chance to get called before "ssl_protocol.connection_made()". transport.pause_reading() + # gh-142352: move buffered StreamReader data to SSLProtocol + if server_side: + from .streams import StreamReaderProtocol + if isinstance(protocol, StreamReaderProtocol): + stream_reader = getattr(protocol, '_stream_reader', None) + if stream_reader is not None: + buffer = stream_reader._buffer + if buffer: + ssl_protocol._incoming.write(buffer) + buffer.clear() + transport.set_protocol(ssl_protocol) conmade_cb = self.call_soon(ssl_protocol.connection_made, transport) resume_cb = self.call_soon(transport.resume_reading) @@ -1532,11 +1550,11 @@ async def create_server( The host parameter can be a string, in that case the TCP server is bound to host and port. - The host parameter can also be a sequence of strings and in that case - the TCP server is bound to all hosts of the sequence. If a host - appears multiple times (possibly indirectly e.g. when hostnames - resolve to the same IP address), the server is only bound once to that - host. + The host parameter can also be a sequence of strings and in that + case the TCP server is bound to all hosts of the sequence. If + a host appears multiple times (possibly indirectly e.g. when + hostnames resolve to the same IP address), the server is only bound + once to that host. Return a Server object which can be used to stop the service. @@ -2011,7 +2029,10 @@ def _run_once(self): event_list = None # Handle 'later' callbacks that are ready. - end_time = self.time() + self._clock_resolution + now = self.time() + # Ensure that `end_time` is strictly increasing + # when the clock resolution is too small. + end_time = now + max(self._clock_resolution, math.ulp(now)) while self._scheduled: handle = self._scheduled[0] if handle._when >= end_time: diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index d40af422e614c15..224b1883808a412 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -26,6 +26,7 @@ def __init__(self, loop, protocol, args, shell, self._pending_calls = collections.deque() self._pipes = {} self._finished = False + self._pipes_connected = False if stdin == subprocess.PIPE: self._pipes[0] = None @@ -213,6 +214,7 @@ async def _connect_pipes(self, waiter): else: if waiter is not None and not waiter.cancelled(): waiter.set_result(None) + self._pipes_connected = True def _call(self, cb, *data): if self._pending_calls is not None: @@ -256,6 +258,15 @@ def _try_finish(self): assert not self._finished if self._returncode is None: return + if not self._pipes_connected: + # self._pipes_connected can be False if not all pipes were connected + # because either the process failed to start or the self._connect_pipes task + # got cancelled. In this broken state we consider all pipes disconnected and + # to avoid hanging forever in self._wait as otherwise _exit_waiters + # would never be woken up, we wake them up here. + for waiter in self._exit_waiters: + if not waiter.done(): + waiter.set_result(self._returncode) if all(p is not None and p.disconnected for p in self._pipes.values()): self._finished = True @@ -267,7 +278,7 @@ def _call_connection_lost(self, exc): finally: # wake up futures waiting for wait() for waiter in self._exit_waiters: - if not waiter.cancelled(): + if not waiter.done(): waiter.set_result(self._returncode) self._exit_waiters = None self._loop = None diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index a51319cb72a6a9b..3a4246f6ccd4c24 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -1,7 +1,6 @@ -__all__ = 'iscoroutinefunction', 'iscoroutine' +__all__ = ('iscoroutine',) import collections.abc -import inspect import os import sys import types @@ -13,25 +12,6 @@ def _is_debug_mode(): bool(os.environ.get('PYTHONASYNCIODEBUG'))) -# A marker for iscoroutinefunction. -_is_coroutine = object() - - -def iscoroutinefunction(func): - import warnings - """Return True if func is a decorated coroutine function.""" - warnings._deprecated("asyncio.iscoroutinefunction", - f"{warnings._DEPRECATED_MSG}; " - "use inspect.iscoroutinefunction() instead", - remove=(3,16)) - return _iscoroutinefunction(func) - - -def _iscoroutinefunction(func): - return (inspect.iscoroutinefunction(func) or - getattr(func, '_is_coroutine', None) is _is_coroutine) - - # Prioritize native coroutine check to speed-up # asyncio.iscoroutine. _COROUTINE_TYPES = (types.CoroutineType, collections.abc.Coroutine) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index a7fb55982abe9ca..42d8408a38dfd1f 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -374,8 +374,8 @@ async def create_server( If host is an empty string or None all interfaces are assumed and a list of multiple sockets will be returned (most likely - one for IPv4 and another one for IPv6). The host parameter can also be - a sequence (e.g. list) of hosts to bind to. + one for IPv4 and another one for IPv6). The host parameter can also + be a sequence (e.g. list) of hosts to bind to. family can be set to either AF_INET or AF_INET6 to force the socket to use IPv4 or IPv6. If not set it will be determined @@ -415,8 +415,9 @@ async def create_server( start_serving set to True (default) causes the created server to start accepting connections immediately. When set to False, - the user should await Server.start_serving() or Server.serve_forever() - to make the server to start accepting connections. + the user should await Server.start_serving() or + Server.serve_forever() to make the server to start accepting + connections. """ raise NotImplementedError @@ -479,8 +480,9 @@ async def create_unix_server( start_serving set to True (default) causes the created server to start accepting connections immediately. When set to False, - the user should await Server.start_serving() or Server.serve_forever() - to make the server to start accepting connections. + the user should await Server.start_serving() or + Server.serve_forever() to make the server to start accepting + connections. """ raise NotImplementedError @@ -511,8 +513,8 @@ async def create_datagram_endpoint(self, protocol_factory, protocol_factory must be a callable returning a protocol instance. - socket family AF_INET, socket.AF_INET6 or socket.AF_UNIX depending on - host (or family if specified), socket type SOCK_DGRAM. + socket family AF_INET, socket.AF_INET6 or socket.AF_UNIX depending + on host (or family if specified), socket type SOCK_DGRAM. reuse_address tells the kernel to reuse a local socket in TIME_WAIT state, without waiting for its natural timeout to @@ -552,7 +554,8 @@ async def connect_read_pipe(self, protocol_factory, pipe): async def connect_write_pipe(self, protocol_factory, pipe): """Register write pipe in event loop. - protocol_factory should instantiate object with BaseProtocol interface. + protocol_factory should instantiate object with BaseProtocol + interface. Pipe is file-like object already switched to nonblocking. Return pair (transport, protocol), where transport support WriteTransport interface.""" diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 6bd00a644789f10..11858a0274a69fd 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -79,6 +79,10 @@ def __init__(self, *, loop=None): loop object used by the future. If it's not provided, the future uses the default event loop. """ + if self._loop is not None: + raise RuntimeError(f"{self.__class__.__name__} object is already " + "initialized") + if loop is None: self._loop = events.get_event_loop() else: @@ -389,7 +393,7 @@ def _set_state(future, other): def _call_check_cancel(destination): if destination.cancelled(): - if source_loop is None or source_loop is dest_loop: + if source_loop is None or source_loop is events._get_running_loop(): source.cancel() else: source_loop.call_soon_threadsafe(source.cancel) @@ -398,7 +402,7 @@ def _call_set_state(source): if (destination.cancelled() and dest_loop is not None and dest_loop.is_closed()): return - if dest_loop is None or dest_loop is source_loop: + if dest_loop is None or dest_loop is events._get_running_loop(): _set_state(destination, source) else: if dest_loop.is_closed(): diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index b5bfeb1630a1590..35f7fa62140bd49 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -112,13 +112,13 @@ def capture_call_graph( optional keyword-only 'depth' argument can be used to skip the specified number of frames from top of the stack. - If the optional keyword-only 'limit' argument is provided, each call stack - in the resulting graph is truncated to include at most ``abs(limit)`` - entries. If 'limit' is positive, the entries left are the closest to - the invocation point. If 'limit' is negative, the topmost entries are - left. If 'limit' is omitted or None, all entries are present. - If 'limit' is 0, the call stack is not captured at all, only - "awaited by" information is present. + If the optional keyword-only 'limit' argument is provided, each call + stack in the resulting graph is truncated to include at most + ``abs(limit)`` entries. If 'limit' is positive, the entries left are + the closest to the invocation point. If 'limit' is negative, the + topmost entries are left. If 'limit' is omitted or None, all entries + are present. If 'limit' is 0, the call stack is not captured at all, + only "awaited by" information is present. """ loop = events._get_running_loop() diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index fa3a94764b507ae..c59ea15111bef2b 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -145,10 +145,7 @@ def _wake_up_first(self): """Ensure that the first waiter will wake up.""" if not self._waiters: return - try: - fut = next(iter(self._waiters)) - except StopIteration: - return + fut = next(iter(self._waiters)) # .done() means that the waiter is already set to wake up. if not fut.done(): @@ -158,10 +155,10 @@ def _wake_up_first(self): class Event(mixins._LoopBoundMixin): """Asynchronous equivalent to threading.Event. - Class implementing event objects. An event manages a flag that can be set - to true with the set() method and reset to false with the clear() method. - The wait() method blocks until the flag is true. The flag is initially - false. + Class implementing event objects. An event manages a flag that can be + set to true with the set() method and reset to false with the clear() + method. The wait() method blocks until the flag is true. The flag is + initially false. """ def __init__(self): @@ -353,9 +350,9 @@ class Semaphore(_ContextManagerMixin, mixins._LoopBoundMixin): """A Semaphore implementation. A semaphore manages an internal counter which is decremented by each - acquire() call and incremented by each release() call. The counter - can never go below zero; when acquire() finds that it is zero, it blocks, - waiting until some other thread calls release(). + acquire() call and incremented by each release() call. The counter + can never go below zero; when acquire() finds that it is zero, it + blocks, waiting until some other thread calls release(). Semaphores also support the context management protocol. @@ -511,8 +508,8 @@ async def __aexit__(self, *args): async def wait(self): """Wait for the barrier. - When the specified number of tasks have started waiting, they are all - simultaneously awoken. + When the specified number of tasks have started waiting, they are + all simultaneously awoken. Returns an unique and individual index number from 0 to 'parties-1'. """ async with self._cond: diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 7eb55bd63ddb73e..cf2902b4c76559e 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -460,6 +460,8 @@ def _pipe_closed(self, fut): class _ProactorDatagramTransport(_ProactorBasePipeTransport, transports.DatagramTransport): max_size = 256 * 1024 + _header_size = 8 + def __init__(self, loop, sock, protocol, address=None, waiter=None, extra=None): self._address = address @@ -499,7 +501,7 @@ def sendto(self, data, addr=None): # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) - self._buffer_size += len(data) + 8 # include header bytes + self._buffer_size += len(data) + self._header_size if self._write_fut is None: # No current write operations are active, kick one off @@ -526,7 +528,7 @@ def _loop_writing(self, fut=None): return data, addr = self._buffer.popleft() - self._buffer_size -= len(data) + self._buffer_size -= len(data) + self._header_size if self._address is not None: self._write_fut = self._loop._proactor.send(self._sock, data) @@ -640,7 +642,7 @@ def __init__(self, proactor): signal.set_wakeup_fd(self._csock.fileno()) def _make_socket_transport(self, sock, protocol, waiter=None, - extra=None, server=None): + extra=None, server=None, context=None): return _ProactorSocketTransport(self, sock, protocol, waiter, extra, server) @@ -649,7 +651,7 @@ def _make_ssl_transport( *, server_side=False, server_hostname=None, extra=None, server=None, ssl_handshake_timeout=None, - ssl_shutdown_timeout=None): + ssl_shutdown_timeout=None, context=None): ssl_protocol = sslproto.SSLProtocol( self, protocol, sslcontext, waiter, server_side, server_hostname, @@ -731,7 +733,7 @@ async def sock_accept(self, sock): async def _sock_sendfile_native(self, sock, file, offset, count): try: fileno = file.fileno() - except (AttributeError, io.UnsupportedOperation) as err: + except (AttributeError, io.UnsupportedOperation): raise exceptions.SendfileNotAvailableError("not a regular file") try: fsize = os.fstat(fileno).st_size @@ -754,8 +756,7 @@ async def _sock_sendfile_native(self, sock, file, offset, count): offset += blocksize total_sent += blocksize finally: - if total_sent > 0: - file.seek(offset) + file.seek(offset) async def _sendfile_native(self, transp, file, offset, count): resume_reading = transp.is_reading() @@ -835,7 +836,7 @@ def _write_to_self(self): def _start_serving(self, protocol_factory, sock, sslcontext=None, server=None, backlog=100, ssl_handshake_timeout=None, - ssl_shutdown_timeout=None): + ssl_shutdown_timeout=None, context=None): def loop(f=None): try: diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index e5d6f2e4b61e174..30004f2bc9bacd2 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -33,11 +33,11 @@ class QueueShutDown(Exception): class Queue(mixins._LoopBoundMixin): """A queue, useful for coordinating producer and consumer coroutines. - If maxsize is less than or equal to zero, the queue size is infinite. If it - is an integer greater than 0, then "await put()" will block when the - queue reaches maxsize, until an item is removed by get(). + If maxsize is less than or equal to zero, the queue size is infinite. + If it is an integer greater than 0, then "await put()" will block when + the queue reaches maxsize, until an item is removed by get(). - Unlike the standard library Queue, you can reliably know this Queue's size + Unlike queue.Queue, you can reliably know this Queue's size with qsize(), since your single-threaded asyncio application won't be interrupted between calling qsize() and doing an operation on the Queue. """ @@ -174,8 +174,8 @@ async def get(self): If queue is empty, wait until an item is available. - Raises QueueShutDown if the queue has been shut down and is empty, or - if the queue has been shut down immediately. + Raises QueueShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. """ while self.empty(): if self._is_shutdown and self.empty(): @@ -203,10 +203,11 @@ async def get(self): def get_nowait(self): """Remove and return an item from the queue. - Return an item if one is immediately available, else raise QueueEmpty. + Return an item if one is immediately available, else raise + QueueEmpty. - Raises QueueShutDown if the queue has been shut down and is empty, or - if the queue has been shut down immediately. + Raises QueueShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. """ if self.empty(): if self._is_shutdown: @@ -223,12 +224,12 @@ def task_done(self): a subsequent call to task_done() tells the queue that the processing on the task is complete. - If a join() is currently blocking, it will resume when all items have - been processed (meaning that a task_done() call was received for every - item that had been put() into the queue). + If a join() is currently blocking, it will resume when all items + have been processed (meaning that a task_done() call was received + for every item that had been put() into the queue). - Raises ValueError if called more times than there were items placed in - the queue. + Raises ValueError if called more times than there were items placed + in the queue. """ if self._unfinished_tasks <= 0: raise ValueError('task_done() called too many times') @@ -239,10 +240,11 @@ def task_done(self): async def join(self): """Block until all items in the queue have been gotten and processed. - The count of unfinished tasks goes up whenever an item is added to the - queue. The count goes down whenever a consumer calls task_done() to - indicate that the item was retrieved and all work on it is complete. - When the count of unfinished tasks drops to zero, join() unblocks. + The count of unfinished tasks goes up whenever an item is added to + the queue. The count goes down whenever a consumer calls + task_done() to indicate that the item was retrieved and all work on + it is complete. When the count of unfinished tasks drops to zero, + join() unblocks. """ if self._unfinished_tasks > 0: await self._finished.wait() @@ -253,9 +255,11 @@ def shutdown(self, immediate=False): By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. - All blocked callers of put() and get() will be unblocked. If - 'immediate', unblock callers of join() regardless of the - number of unfinished tasks. + All blocked callers of put() and get() will be unblocked. + + If 'immediate', the queue is drained and unfinished tasks + is reduced by the number of drained tasks. If unfinished tasks + is reduced to zero, callers of Queue.join are unblocked. """ self._is_shutdown = True if immediate: diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index ba37e003a655c0a..774a317564df22c 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -35,7 +35,8 @@ class Runner: with asyncio.Runner(debug=True) as runner: runner.run(main()) - The run() method can be called multiple times within the runner's context. + The run() method can be called multiple times within the runner's + context. This can be useful for interactive console (e.g. IPython), unittest runners, console tools, -- everywhere when async code diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 6ad84044adf1463..4fa4dd5a885236e 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -53,7 +53,7 @@ def _test_selector_event(selector, fd, event): class BaseSelectorEventLoop(base_events.BaseEventLoop): """Selector event loop. - See events.EventLoop for API specification. + See events.AbstractEventLoop for API specification. """ def __init__(self, selector=None): @@ -67,10 +67,10 @@ def __init__(self, selector=None): self._transports = weakref.WeakValueDictionary() def _make_socket_transport(self, sock, protocol, waiter=None, *, - extra=None, server=None): + extra=None, server=None, context=None): self._ensure_fd_no_transport(sock) return _SelectorSocketTransport(self, sock, protocol, waiter, - extra, server) + extra, server, context=context) def _make_ssl_transport( self, rawsock, protocol, sslcontext, waiter=None, @@ -78,16 +78,17 @@ def _make_ssl_transport( extra=None, server=None, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, + context=None, ): self._ensure_fd_no_transport(rawsock) ssl_protocol = sslproto.SSLProtocol( self, protocol, sslcontext, waiter, server_side, server_hostname, ssl_handshake_timeout=ssl_handshake_timeout, - ssl_shutdown_timeout=ssl_shutdown_timeout + ssl_shutdown_timeout=ssl_shutdown_timeout, ) _SelectorSocketTransport(self, rawsock, ssl_protocol, - extra=extra, server=server) + extra=extra, server=server, context=context) return ssl_protocol._app_transport def _make_datagram_transport(self, sock, protocol, @@ -159,16 +160,16 @@ def _write_to_self(self): def _start_serving(self, protocol_factory, sock, sslcontext=None, server=None, backlog=100, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, - ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT): + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, context=None): self._add_reader(sock.fileno(), self._accept_connection, protocol_factory, sock, sslcontext, server, backlog, - ssl_handshake_timeout, ssl_shutdown_timeout) + ssl_handshake_timeout, ssl_shutdown_timeout, context) def _accept_connection( self, protocol_factory, sock, sslcontext=None, server=None, backlog=100, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, - ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT): + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, context=None): # This method is only called once for each event loop tick where the # listening socket has triggered an EVENT_READ. There may be multiple # connections waiting for an .accept() so it is called in a loop. @@ -204,21 +205,22 @@ def _accept_connection( self._start_serving, protocol_factory, sock, sslcontext, server, backlog, ssl_handshake_timeout, - ssl_shutdown_timeout) + ssl_shutdown_timeout, context) else: raise # The event loop will catch, log and ignore it. else: extra = {'peername': addr} + conn_context = context.copy() if context is not None else None accept = self._accept_connection2( protocol_factory, conn, extra, sslcontext, server, - ssl_handshake_timeout, ssl_shutdown_timeout) - self.create_task(accept) + ssl_handshake_timeout, ssl_shutdown_timeout, context=conn_context) + self.create_task(accept, context=conn_context) async def _accept_connection2( self, protocol_factory, conn, extra, sslcontext=None, server=None, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, - ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT): + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, context=None): protocol = None transport = None try: @@ -229,11 +231,12 @@ async def _accept_connection2( conn, protocol, sslcontext, waiter=waiter, server_side=True, extra=extra, server=server, ssl_handshake_timeout=ssl_handshake_timeout, - ssl_shutdown_timeout=ssl_shutdown_timeout) + ssl_shutdown_timeout=ssl_shutdown_timeout, + context=context) else: transport = self._make_socket_transport( conn, protocol, waiter=waiter, extra=extra, - server=server) + server=server, context=context) try: await waiter @@ -275,9 +278,9 @@ def _ensure_fd_no_transport(self, fd): f'File descriptor {fd!r} is used by transport ' f'{transport!r}') - def _add_reader(self, fd, callback, *args): + def _add_reader(self, fd, callback, *args, context=None): self._check_closed() - handle = events.Handle(callback, args, self, None) + handle = events.Handle(callback, args, self, context=context) key = self._selector.get_map().get(fd) if key is None: self._selector.register(fd, selectors.EVENT_READ, @@ -309,9 +312,9 @@ def _remove_reader(self, fd): else: return False - def _add_writer(self, fd, callback, *args): + def _add_writer(self, fd, callback, *args, context=None): self._check_closed() - handle = events.Handle(callback, args, self, None) + handle = events.Handle(callback, args, self, context=context) key = self._selector.get_map().get(fd) if key is None: self._selector.register(fd, selectors.EVENT_WRITE, @@ -530,11 +533,12 @@ def _sock_recvfrom_into(self, fut, sock, buf, bufsize): async def sock_sendall(self, sock, data): """Send data to the socket. - The socket must be connected to a remote socket. This method continues - to send data from data until either all data has been sent or an - error occurs. None is returned on success. On error, an exception is - raised, and there is no way to determine how much data, if any, was - successfully processed by the receiving end of the connection. + The socket must be connected to a remote socket. This method + continues to send data from data until either all data has been + sent or an error occurs. None is returned on success. On error, + an exception is raised, and there is no way to determine how much + data, if any, was successfully processed by the receiving end of + the connection. """ base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: @@ -583,11 +587,12 @@ def _sock_sendall(self, fut, sock, view, pos): async def sock_sendto(self, sock, data, address): """Send data to the socket. - The socket must be connected to a remote socket. This method continues - to send data from data until either all data has been sent or an - error occurs. None is returned on success. On error, an exception is - raised, and there is no way to determine how much data, if any, was - successfully processed by the receiving end of the connection. + The socket must be connected to a remote socket. This method + continues to send data from data until either all data has been + sent or an error occurs. None is returned on success. On error, + an exception is raised, and there is no way to determine how much + data, if any, was successfully processed by the receiving end of + the connection. """ base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: @@ -698,10 +703,11 @@ def _sock_connect_cb(self, fut, sock, address): async def sock_accept(self, sock): """Accept a connection. - The socket must be bound to an address and listening for connections. - The return value is a pair (conn, address) where conn is a new socket - object usable to send and receive data on the connection, and address - is the address bound to the socket on the other end of the connection. + The socket must be bound to an address and listening for + connections. The return value is a pair (conn, address) where + conn is a new socket object usable to send and receive data on the + connection, and address is the address bound to the socket on the + other end of the connection. """ base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: @@ -770,7 +776,7 @@ class _SelectorTransport(transports._FlowControlMixin, # exception) _sock = None - def __init__(self, loop, sock, protocol, extra=None, server=None): + def __init__(self, loop, sock, protocol, extra=None, server=None, context=None): super().__init__(extra, loop) self._extra['socket'] = trsock.TransportSocket(sock) try: @@ -784,12 +790,13 @@ def __init__(self, loop, sock, protocol, extra=None, server=None): self._extra['peername'] = None self._sock = sock self._sock_fd = sock.fileno() - + self._context = context self._protocol_connected = False self.set_protocol(protocol) self._server = server self._buffer = collections.deque() + self._buffer_size = 0 self._conn_lost = 0 # Set when call to connection_lost scheduled. self._closing = False # Set when close() called. self._paused = False # Set when pause_reading() called @@ -866,7 +873,7 @@ def close(self): if not self._buffer: self._conn_lost += 1 self._loop._remove_writer(self._sock_fd) - self._loop.call_soon(self._call_connection_lost, None) + self._call_soon(self._call_connection_lost, None) def __del__(self, _warn=warnings.warn): if self._sock is not None: @@ -894,12 +901,13 @@ def _force_close(self, exc): return if self._buffer: self._buffer.clear() + self._buffer_size = 0 self._loop._remove_writer(self._sock_fd) if not self._closing: self._closing = True self._loop._remove_reader(self._sock_fd) self._conn_lost += 1 - self._loop.call_soon(self._call_connection_lost, exc) + self._call_soon(self._call_connection_lost, exc) def _call_connection_lost(self, exc): try: @@ -916,13 +924,18 @@ def _call_connection_lost(self, exc): self._server = None def get_write_buffer_size(self): - return sum(map(len, self._buffer)) + return self._buffer_size def _add_reader(self, fd, callback, *args): if not self.is_reading(): return - self._loop._add_reader(fd, callback, *args) + self._loop._add_reader(fd, callback, *args, context=self._context) + def _add_writer(self, fd, callback, *args): + self._loop._add_writer(fd, callback, *args, context=self._context) + + def _call_soon(self, callback, *args): + self._loop.call_soon(callback, *args, context=self._context) class _SelectorSocketTransport(_SelectorTransport): @@ -930,10 +943,9 @@ class _SelectorSocketTransport(_SelectorTransport): _sendfile_compatible = constants._SendfileMode.TRY_NATIVE def __init__(self, loop, sock, protocol, waiter=None, - extra=None, server=None): - + extra=None, server=None, context=None): self._read_ready_cb = None - super().__init__(loop, sock, protocol, extra, server) + super().__init__(loop, sock, protocol, extra, server, context) self._eof = False self._empty_waiter = None if _HAS_SENDMSG: @@ -945,14 +957,12 @@ def __init__(self, loop, sock, protocol, waiter=None, # decreases the latency (in some cases significantly.) base_events._set_nodelay(self._sock) - self._loop.call_soon(self._protocol.connection_made, self) + self._call_soon(self._protocol.connection_made, self) # only start reading when connection_made() has been called - self._loop.call_soon(self._add_reader, - self._sock_fd, self._read_ready) + self._call_soon(self._add_reader, self._sock_fd, self._read_ready) if waiter is not None: # only wake up the waiter when connection_made() has been called - self._loop.call_soon(futures._set_result_unless_cancelled, - waiter, None) + self._call_soon(futures._set_result_unless_cancelled, waiter, None) def set_protocol(self, protocol): if isinstance(protocol, protocols.BufferedProtocol): @@ -1050,8 +1060,8 @@ def _read_ready__on_eof(self): def write(self, data): if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError(f'data argument must be a bytes-like object, ' - f'not {type(data).__name__!r}') + raise TypeError(f'data argument must be a bytes, bytearray, or memoryview ' + f'object, not {type(data).__name__!r}') if self._eof: raise RuntimeError('Cannot call write() after write_eof()') if self._empty_waiter is not None: @@ -1081,10 +1091,11 @@ def write(self, data): if not data: return # Not all was written; register write handler. - self._loop._add_writer(self._sock_fd, self._write_ready) + self._add_writer(self._sock_fd, self._write_ready) # Add it to the buffer. self._buffer.append(data) + self._buffer_size += len(data) self._maybe_pause_protocol() def _get_sendmsg_buffer(self): @@ -1104,6 +1115,7 @@ def _write_sendmsg(self): except BaseException as exc: self._loop._remove_writer(self._sock_fd) self._buffer.clear() + self._buffer_size = 0 self._fatal_error(exc, 'Fatal write error on socket transport') if self._empty_waiter is not None: self._empty_waiter.set_exception(exc) @@ -1119,6 +1131,7 @@ def _write_sendmsg(self): self._sock.shutdown(socket.SHUT_WR) def _adjust_leftover_buffer(self, nbytes: int) -> None: + self._buffer_size -= nbytes buffer = self._buffer while nbytes: b = buffer.popleft() @@ -1139,13 +1152,16 @@ def _write_send(self): if n != len(buffer): # Not all data was written self._buffer.appendleft(buffer[n:]) + self._buffer_size -= n except (BlockingIOError, InterruptedError): - pass + self._buffer.appendleft(buffer) + return except (SystemExit, KeyboardInterrupt): raise except BaseException as exc: self._loop._remove_writer(self._sock_fd) self._buffer.clear() + self._buffer_size = 0 self._fatal_error(exc, 'Fatal write error on socket transport') if self._empty_waiter is not None: self._empty_waiter.set_exception(exc) @@ -1174,11 +1190,20 @@ def writelines(self, list_of_data): raise RuntimeError('unable to writelines; sendfile is in progress') if not list_of_data: return - self._buffer.extend([memoryview(data) for data in list_of_data]) + + if self._conn_lost: + if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: + logger.warning('socket.send() raised exception.') + self._conn_lost += 1 + return + + for data in list_of_data: + self._buffer.append(memoryview(data)) + self._buffer_size += len(data) self._write_ready() # If the entire buffer couldn't be written, register a write handler if self._buffer: - self._loop._add_writer(self._sock_fd, self._write_ready) + self._add_writer(self._sock_fd, self._write_ready) self._maybe_pause_protocol() def can_write_eof(self): @@ -1211,21 +1236,19 @@ def close(self): class _SelectorDatagramTransport(_SelectorTransport, transports.DatagramTransport): - _buffer_factory = collections.deque + _header_size = 8 def __init__(self, loop, sock, protocol, address=None, waiter=None, extra=None): super().__init__(loop, sock, protocol, extra) self._address = address self._buffer_size = 0 - self._loop.call_soon(self._protocol.connection_made, self) + self._call_soon(self._protocol.connection_made, self) # only start reading when connection_made() has been called - self._loop.call_soon(self._add_reader, - self._sock_fd, self._read_ready) + self._call_soon(self._add_reader, self._sock_fd, self._read_ready) if waiter is not None: # only wake up the waiter when connection_made() has been called - self._loop.call_soon(futures._set_result_unless_cancelled, - waiter, None) + self._call_soon(futures._set_result_unless_cancelled, waiter, None) def get_write_buffer_size(self): return self._buffer_size @@ -1272,7 +1295,7 @@ def sendto(self, data, addr=None): self._sock.sendto(data, addr) return except (BlockingIOError, InterruptedError): - self._loop._add_writer(self._sock_fd, self._sendto_ready) + self._add_writer(self._sock_fd, self._sendto_ready) except OSError as exc: self._protocol.error_received(exc) return @@ -1285,13 +1308,13 @@ def sendto(self, data, addr=None): # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) - self._buffer_size += len(data) + 8 # include header bytes + self._buffer_size += len(data) + self._header_size self._maybe_pause_protocol() def _sendto_ready(self): while self._buffer: data, addr = self._buffer.popleft() - self._buffer_size -= len(data) + self._buffer_size -= len(data) + self._header_size try: if self._extra['peername']: self._sock.send(data) @@ -1299,7 +1322,7 @@ def _sendto_ready(self): self._sock.sendto(data, addr) except (BlockingIOError, InterruptedError): self._buffer.appendleft((data, addr)) # Try again later. - self._buffer_size += len(data) + self._buffer_size += len(data) + self._header_size break except OSError as exc: self._protocol.error_received(exc) diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index 2ad65d8648e6c52..845aed4c6a3b352 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -62,7 +62,6 @@ async def staggered_race(coro_fns, delay, *, loop=None): coroutine's entry is ``None``. """ - # TODO: when we have aiter() and anext(), allow async iterables in coro_fns. loop = loop or events.get_running_loop() parent_task = tasks.current_task(loop) enum_coro_fns = enumerate(coro_fns) diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 64aac4cc50d15a5..a28c11e928f806a 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -214,7 +214,6 @@ def _stream_reader(self): return self._stream_reader_wr() def _replace_transport(self, transport): - loop = self._loop self._transport = transport self._over_ssl = transport.get_extra_info('sslcontext') is not None @@ -271,7 +270,6 @@ def connection_lost(self, exc): self._closed.set_exception(exc) super().connection_lost(exc) self._stream_reader_wr = None - self._stream_writer = None self._task = None self._transport = None @@ -541,17 +539,17 @@ async def _wait_for_data(self, func_name): self._waiter = None async def readline(self): - """Read chunk of data from the stream until newline (b'\n') is found. + r"""Read chunk of data from the stream until newline (b'\n') is found. - On success, return chunk that ends with newline. If only partial + On success, return chunk that ends with newline. If only partial line can be read due to EOF, return incomplete line without - terminating newline. When EOF was reached while no bytes read, empty - bytes object is returned. + terminating newline. When EOF was reached while no bytes read, + empty bytes object is returned. - If limit is reached, ValueError will be raised. In that case, if + If limit is reached, ValueError will be raised. In that case, if newline was found, complete line including newline will be removed - from internal buffer. Else, internal buffer will be cleared. Limit is - compared against part of the line without newline. + from internal buffer. Else, internal buffer will be cleared. + Limit is compared against part of the line without newline. If stream was paused, this function will automatically resume it if needed. @@ -669,8 +667,7 @@ async def readuntil(self, separator=b'\n'): # adds data which makes separator be found. That's why we check for # EOF *after* inspecting the buffer. if self._eof: - chunk = bytes(self._buffer) - self._buffer.clear() + chunk = self._buffer.take_bytes() raise exceptions.IncompleteReadError(chunk, None) # _wait_for_data() will resume reading if stream was paused. @@ -680,10 +677,9 @@ async def readuntil(self, separator=b'\n'): raise exceptions.LimitOverrunError( 'Separator is found, but chunk is longer than limit', match_start) - chunk = self._buffer[:match_end] - del self._buffer[:match_end] + chunk = self._buffer.take_bytes(match_end) self._maybe_resume_transport() - return bytes(chunk) + return chunk async def read(self, n=-1): """Read up to `n` bytes from the stream. @@ -718,20 +714,16 @@ async def read(self, n=-1): # collect everything in self._buffer, but that would # deadlock if the subprocess sends more than self.limit # bytes. So just call self.read(self._limit) until EOF. - blocks = [] - while True: - block = await self.read(self._limit) - if not block: - break - blocks.append(block) - return b''.join(blocks) + joined = bytearray() + while block := await self.read(self._limit): + joined += block + return joined.take_bytes() if not self._buffer and not self._eof: await self._wait_for_data('read') # This will work right even if buffer is less than n bytes - data = bytes(memoryview(self._buffer)[:n]) - del self._buffer[:n] + data = self._buffer.take_bytes(min(len(self._buffer), n)) self._maybe_resume_transport() return data @@ -762,18 +754,12 @@ async def readexactly(self, n): while len(self._buffer) < n: if self._eof: - incomplete = bytes(self._buffer) - self._buffer.clear() + incomplete = self._buffer.take_bytes() raise exceptions.IncompleteReadError(incomplete, n) await self._wait_for_data('readexactly') - if len(self._buffer) == n: - data = bytes(self._buffer) - self._buffer.clear() - else: - data = bytes(memoryview(self._buffer)[:n]) - del self._buffer[:n] + data = self._buffer.take_bytes(n) self._maybe_resume_transport() return data diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 00e8f6d5d1a68b2..e1ec025791a52e6 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -37,6 +37,7 @@ def __init__(self): self._errors = [] self._base_error = None self._on_completed_fut = None + self._cancel_on_enter = False def __repr__(self): info = [''] @@ -63,6 +64,8 @@ async def __aenter__(self): raise RuntimeError( f'TaskGroup {self!r} cannot determine the parent task') self._entered = True + if self._cancel_on_enter: + self.cancel() return self @@ -178,6 +181,9 @@ async def _aexit(self, et, exc): finally: exc = None + # Suppress any remaining exception (exceptions deserving to be raised + # were raised above). + return True def create_task(self, coro, **kwargs): """Create a new task in this group and return it. @@ -278,3 +284,30 @@ def _on_task_done(self, task): self._abort() self._parent_cancel_requested = True self._parent_task.cancel() + + def cancel(self): + """Cancel the task group + + `cancel()` will be called on any tasks in the group that aren't yet + done, as well as the parent (body) of the group. This will cause + the task group context manager to exit *without* + `asyncio.CancelledError` being raised. + + If `cancel()` is called before entering the task group, the group + will be cancelled upon entry. This is useful for patterns where + one piece of code passes an unused TaskGroup instance to another in + order to have the ability to cancel anything run within the group. + + `cancel()` is idempotent and may be called after the task group has + already exited. + """ + if not self._entered: + self._cancel_on_enter = True + return + if self._exiting and not self._tasks: + return + if not self._aborting: + self._abort() + if self._parent_task and not self._parent_cancel_requested: + self._parent_cancel_requested = True + self._parent_task.cancel() diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index fbd5c39a7c56ac4..14d3c1ceb58cacb 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -624,10 +624,11 @@ def as_completed(fs, *, timeout=None): Run the supplied awaitables concurrently. The returned object can be iterated to obtain the results of the awaitables as they finish. - The object returned can be iterated as an asynchronous iterator or a plain - iterator. When asynchronous iteration is used, the originally-supplied - awaitables are yielded if they are tasks or futures. This makes it easy to - correlate previously-scheduled tasks with their results: + The object returned can be iterated as an asynchronous iterator or + a plain iterator. When asynchronous iteration is used, the + originally-supplied awaitables are yielded if they are tasks or + futures. This makes it easy to correlate previously-scheduled tasks + with their results: ipv4_connect = create_task(open_connection("127.0.0.1", 80)) ipv6_connect = create_task(open_connection("::1", 80)) @@ -643,26 +644,27 @@ def as_completed(fs, *, timeout=None): else: print("IPv4 connection established.") - During asynchronous iteration, implicitly-created tasks will be yielded for - supplied awaitables that aren't tasks or futures. + During asynchronous iteration, implicitly-created tasks will be + yielded for supplied awaitables that aren't tasks or futures. - When used as a plain iterator, each iteration yields a new coroutine that - returns the result or raises the exception of the next completed awaitable. - This pattern is compatible with Python versions older than 3.13: + When used as a plain iterator, each iteration yields a new coroutine + that returns the result or raises the exception of the next completed + awaitable. This pattern is compatible with Python versions older than + 3.13: ipv4_connect = create_task(open_connection("127.0.0.1", 80)) ipv6_connect = create_task(open_connection("::1", 80)) tasks = [ipv4_connect, ipv6_connect] for next_connect in as_completed(tasks): - # next_connect is not one of the original task objects. It must be - # awaited to obtain the result value or raise the exception of the - # awaitable that finishes next. + # next_connect is not one of the original task objects. It must + # be awaited to obtain the result value or raise the exception + # of the awaitable that finishes next. reader, writer = await next_connect - A TimeoutError is raised if the timeout occurs before all awaitables are - done. This is raised by the async for loop during asynchronous iteration or - by the coroutines yielded during plain iteration. + A TimeoutError is raised if the timeout occurs before all awaitables + are done. This is raised by the async for loop during asynchronous + iteration or by the coroutines yielded during plain iteration. """ if inspect.isawaitable(fs): raise TypeError( @@ -1030,21 +1032,22 @@ def callback(): def create_eager_task_factory(custom_task_constructor): """Create a function suitable for use as a task factory on an event-loop. - Example usage: + Example usage: - loop.set_task_factory( - asyncio.create_eager_task_factory(my_task_constructor)) + loop.set_task_factory( + asyncio.create_eager_task_factory(my_task_constructor)) - Now, tasks created will be started immediately (rather than being first - scheduled to an event loop). The constructor argument can be any callable - that returns a Task-compatible object and has a signature compatible - with `Task.__init__`; it must have the `eager_start` keyword argument. + Now, tasks created will be started immediately (rather than being first + scheduled to an event loop). The constructor argument can be any + callable that returns a Task-compatible object and has a signature + compatible with `Task.__init__`; it must have the `eager_start` + keyword argument. - Most applications will use `Task` for `custom_task_constructor` and in - this case there's no need to call `create_eager_task_factory()` - directly. Instead the global `eager_task_factory` instance can be - used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. - """ + Most applications will use `Task` for `custom_task_constructor` and in + this case there's no need to call `create_eager_task_factory()` + directly. Instead the global `eager_task_factory` instance can be + used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. + """ def factory(loop, coro, *, eager_start=True, **kwargs): return custom_task_constructor( diff --git a/Lib/asyncio/threads.py b/Lib/asyncio/threads.py index db048a8231de166..5001351b0976ecd 100644 --- a/Lib/asyncio/threads.py +++ b/Lib/asyncio/threads.py @@ -17,7 +17,8 @@ async def to_thread(func, /, *args, **kwargs): allowing context variables from the main thread to be accessed in the separate thread. - Return a coroutine that can be awaited to get the eventual result of *func*. + Return a coroutine that can be awaited to get the eventual result of + *func*. """ loop = events.get_running_loop() ctx = contextvars.copy_context() diff --git a/Lib/asyncio/timeouts.py b/Lib/asyncio/timeouts.py index 09342dc7c1310b0..65ddc285abd971d 100644 --- a/Lib/asyncio/timeouts.py +++ b/Lib/asyncio/timeouts.py @@ -25,7 +25,8 @@ class _State(enum.Enum): class Timeout: """Asynchronous context manager for cancelling overdue coroutines. - Use `timeout()` or `timeout_at()` rather than instantiating this class directly. + Use `timeout()` or `timeout_at()` rather than instantiating this class + directly. """ def __init__(self, when: float | None) -> None: diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 2683f34cc7113b5..2ac1738d15c6c72 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -1,6 +1,6 @@ """Tools to analyze tasks running in asyncio programs.""" -from collections import defaultdict, namedtuple +from collections import defaultdict from itertools import count from enum import Enum import sys @@ -27,10 +27,10 @@ def __init__( # ─── indexing helpers ─────────────────────────────────────────── def _format_stack_entry(elem: str|FrameInfo) -> str: if not isinstance(elem, str): - if elem.lineno == 0 and elem.filename == "": + if elem.location.lineno == 0 and elem.filename == "": return f"{elem.funcname}" else: - return f"{elem.funcname} {elem.filename}:{elem.lineno}" + return f"{elem.funcname} {elem.filename}:{elem.location.lineno}" return elem @@ -222,20 +222,47 @@ def _print_cycle_exception(exception: CycleFoundException): print(f"cycle: {inames}", file=sys.stderr) -def _get_awaited_by_tasks(pid: int) -> list: - try: - return get_all_awaited_by(pid) - except RuntimeError as e: - while e.__context__ is not None: - e = e.__context__ - print(f"Error retrieving tasks: {e}") - sys.exit(1) +def exit_with_permission_help_text(): + """ + Prints a message pointing to platform-specific permission help text and exits the program. + This function is called when a PermissionError is encountered while trying + to attach to a process. + """ + print( + "Error: The specified process cannot be attached to due to insufficient permissions.\n" + "See the Python documentation for details on required privileges and troubleshooting:\n" + "https://docs.python.org/3/howto/remote_debugging.html#permission-requirements\n", + file=sys.stderr, + ) + sys.exit(1) + + +_TRANSIENT_ERRORS = (RuntimeError, OSError, UnicodeDecodeError, MemoryError) + + +def _get_awaited_by_tasks(pid: int, retries: int = 3) -> list: + for attempt in range(retries + 1): + try: + return get_all_awaited_by(pid) + except PermissionError: + exit_with_permission_help_text() + except ProcessLookupError: + print(f"Error: process {pid} not found.", file=sys.stderr) + sys.exit(1) + except _TRANSIENT_ERRORS as e: + if attempt < retries: + continue + if isinstance(e, RuntimeError): + while e.__context__ is not None: + e = e.__context__ + print(f"Error retrieving tasks: {e}", file=sys.stderr) + sys.exit(1) -def display_awaited_by_tasks_table(pid: int) -> None: +def display_awaited_by_tasks_table(pid: int, retries: int = 3) -> None: """Build and print a table of all pending tasks under `pid`.""" - tasks = _get_awaited_by_tasks(pid) + tasks = _get_awaited_by_tasks(pid, retries=retries) table = build_task_table(tasks) # Print the table in a simple tabular format print( @@ -246,10 +273,10 @@ def display_awaited_by_tasks_table(pid: int) -> None: print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<50} {row[5]:<15} {row[6]:<15}") -def display_awaited_by_tasks_tree(pid: int) -> None: +def display_awaited_by_tasks_tree(pid: int, retries: int = 3) -> None: """Build and print a tree of all pending tasks under `pid`.""" - tasks = _get_awaited_by_tasks(pid) + tasks = _get_awaited_by_tasks(pid, retries=retries) try: result = build_async_tree(tasks) except CycleFoundException as e: diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 1c1458127db5ac1..676e0b670faa49b 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -12,6 +12,7 @@ import sys import threading import warnings +import inspect from . import base_events from . import base_subprocess @@ -54,7 +55,8 @@ def waitstatus_to_exitcode(status): class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): """Unix event loop. - Adds signal handling and UNIX Domain Socket support to SelectorEventLoop. + Adds signal handling and UNIX Domain Socket support to + SelectorEventLoop. """ def __init__(self, selector=None): @@ -94,7 +96,7 @@ def add_signal_handler(self, sig, callback, *args): Raise RuntimeError if there is a problem setting up the handler. """ if (coroutines.iscoroutine(callback) or - coroutines._iscoroutinefunction(callback)): + inspect.iscoroutinefunction(callback)): raise TypeError("coroutines cannot be used " "with add_signal_handler()") self._check_signal(sig) @@ -359,7 +361,7 @@ async def _sock_sendfile_native(self, sock, file, offset, count): "os.sendfile() is not available") try: fileno = file.fileno() - except (AttributeError, io.UnsupportedOperation) as err: + except (AttributeError, io.UnsupportedOperation): raise exceptions.SendfileNotAvailableError("not a regular file") try: fsize = os.fstat(fileno).st_size @@ -384,12 +386,12 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, # order to simplify the common case. self.remove_writer(registered_fd) if fut.cancelled(): - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) return if count: blocksize = count - total_sent if blocksize <= 0: - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_result(total_sent) return @@ -423,20 +425,20 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, # plain send(). err = exceptions.SendfileNotAvailableError( "os.sendfile call failed") - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_exception(err) else: - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_exception(exc) except (SystemExit, KeyboardInterrupt): raise except BaseException as exc: - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_exception(exc) else: if sent == 0: # EOF - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_result(total_sent) else: offset += sent @@ -447,9 +449,9 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, fd, sock, fileno, offset, count, blocksize, total_sent) - def _sock_sendfile_update_filepos(self, fileno, offset, total_sent): - if total_sent > 0: - os.lseek(fileno, offset, os.SEEK_SET) + def _sock_sendfile_update_filepos(self, fileno, offset): + # After this helper runs, the source fd's lseek pointer is at offset." + os.lseek(fileno, offset, os.SEEK_SET) def _sock_add_cancellation_callback(self, fut, sock): def cb(fut): @@ -834,7 +836,8 @@ class _UnixSubprocessTransport(base_subprocess.BaseSubprocessTransport): def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): stdin_w = None - if stdin == subprocess.PIPE and sys.platform.startswith('aix'): + if (stdin == subprocess.PIPE + and (sys.platform.startswith('aix') or sys.platform == 'cygwin')): # Use a socket pair for stdin on AIX, since it does not # support selecting read events on the write end of a # socket (which we use in order to detect closing of the @@ -886,8 +889,8 @@ def _do_wait(self, pid, pidfd, callback, args): pid) else: returncode = waitstatus_to_exitcode(status) - - os.close(pidfd) + finally: + os.close(pidfd) callback(pid, returncode, *args) class _ThreadedChildWatcher: diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index 5f75b17d8ca649b..0bf7732136f1f8e 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -610,6 +610,9 @@ def sendfile(self, sock, file, offset, count): ov = _overlapped.Overlapped(NULL) offset_low = offset & 0xffff_ffff offset_high = (offset >> 32) & 0xffff_ffff + # TransmitFile ignores OVERLAPPED.Offset for handles not opened with + # FILE_FLAG_OVERLAPPED, so seek the CRT file pointer to match. + file.seek(offset) ov.TransmitFile(sock.fileno(), msvcrt.get_osfhandle(file.fileno()), offset_low, offset_high, diff --git a/Lib/asyncio/windows_utils.py b/Lib/asyncio/windows_utils.py index ef277fac3e291c3..d6393f0b1ffee5d 100644 --- a/Lib/asyncio/windows_utils.py +++ b/Lib/asyncio/windows_utils.py @@ -10,7 +10,6 @@ import msvcrt import os import subprocess -import tempfile import warnings @@ -24,6 +23,7 @@ PIPE = subprocess.PIPE STDOUT = subprocess.STDOUT _mmap_counter = itertools.count() +_MAX_PIPE_ATTEMPTS = 20 # Replacement for os.pipe() using handles instead of fds @@ -31,10 +31,6 @@ def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE): """Like os.pipe() but with overlapped support and using handles not fds.""" - address = tempfile.mktemp( - prefix=r'\\.\pipe\python-pipe-{:d}-{:d}-'.format( - os.getpid(), next(_mmap_counter))) - if duplex: openmode = _winapi.PIPE_ACCESS_DUPLEX access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE @@ -56,9 +52,20 @@ def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE): h1 = h2 = None try: - h1 = _winapi.CreateNamedPipe( - address, openmode, _winapi.PIPE_WAIT, - 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL) + for attempts in itertools.count(): + address = r'\\.\pipe\python-pipe-{:d}-{:d}-{}'.format( + os.getpid(), next(_mmap_counter), os.urandom(8).hex()) + try: + h1 = _winapi.CreateNamedPipe( + address, openmode, _winapi.PIPE_WAIT, + 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL) + break + except OSError as e: + if attempts >= _MAX_PIPE_ATTEMPTS: + raise + if e.winerror not in (_winapi.ERROR_PIPE_BUSY, + _winapi.ERROR_ACCESS_DENIED): + raise h2 = _winapi.CreateFile( address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING, @@ -104,8 +111,9 @@ def fileno(self): def close(self, *, CloseHandle=_winapi.CloseHandle): if self._handle is not None: - CloseHandle(self._handle) + handle = self._handle self._handle = None + CloseHandle(handle) def __del__(self, _warn=warnings.warn): if self._handle is not None: diff --git a/Lib/base64.py b/Lib/base64.py index 5d78cc09f40cd3e..fa562f74a810345 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -1,10 +1,9 @@ -"""Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings""" +"""Base16, Base32, Base64 (RFC 4648), Base85 and Ascii85 data encodings""" # Modified 04-Oct-1995 by Jack Jansen to use binascii module # Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support # Modified 22-May-2007 by Guido van Rossum to use bytes everywhere -import struct import binascii @@ -26,6 +25,8 @@ ] +_NOT_SPECIFIED = ['NOT SPECIFIED'] + bytes_types = (bytes, bytearray) # Types acceptable as binary data def _bytes_from_decode_data(s): @@ -45,44 +46,90 @@ def _bytes_from_decode_data(s): # Base64 encoding/decoding uses binascii -def b64encode(s, altchars=None): +def b64encode(s, altchars=None, *, padded=True, wrapcol=0): """Encode the bytes-like object s using Base64 and return a bytes object. Optional altchars should be a byte string of length 2 which specifies an alternative alphabet for the '+' and '/' characters. This allows an application to e.g. generate url or filesystem safe Base64 strings. + + If padded is false, omit padding in the output. + + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. """ - encoded = binascii.b2a_base64(s, newline=False) if altchars is not None: - assert len(altchars) == 2, repr(altchars) - return encoded.translate(bytes.maketrans(b'+/', altchars)) - return encoded + if len(altchars) != 2: + raise ValueError(f'invalid altchars: {altchars!r}') + alphabet = binascii.BASE64_ALPHABET[:-2] + altchars + return binascii.b2a_base64(s, padded=padded, wrapcol=wrapcol, newline=False, + alphabet=alphabet) + return binascii.b2a_base64(s, padded=padded, wrapcol=wrapcol, newline=False) -def b64decode(s, altchars=None, validate=False): +def b64decode(s, altchars=None, validate=_NOT_SPECIFIED, + *, padded=True, ignorechars=_NOT_SPECIFIED, canonical=False): """Decode the Base64 encoded bytes-like object or ASCII string s. Optional altchars must be a bytes-like object or ASCII string of length 2 which specifies the alternative alphabet used instead of the '+' and '/' characters. + If padded is false, padding in input is not required. + The result is returned as a bytes object. A binascii.Error is raised if s is incorrectly padded. - If validate is False (the default), characters that are neither in the - normal base-64 alphabet nor the alternative alphabet are discarded prior - to the padding check. If validate is True, these non-alphabet characters - in the input result in a binascii.Error. + If ignorechars is specified, it should be a byte string containing + characters to ignore from the input. The default value of validate is + True if ignorechars is specified, False otherwise. + + If validate is false, characters that are neither in the normal base-64 + alphabet nor the alternative alphabet are discarded prior to the + padding check. If validate is true, these non-alphabet characters in + the input result in a binascii.Error if they are not in ignorechars. For more information about the strict base64 check, see: https://docs.python.org/3.11/library/binascii.html#binascii.a2b_base64 """ s = _bytes_from_decode_data(s) + if validate is _NOT_SPECIFIED: + validate = ignorechars is not _NOT_SPECIFIED + badchar = None if altchars is not None: altchars = _bytes_from_decode_data(altchars) - assert len(altchars) == 2, repr(altchars) - s = s.translate(bytes.maketrans(altchars, b'+/')) - return binascii.a2b_base64(s, strict_mode=validate) + if len(altchars) != 2: + raise ValueError(f'invalid altchars: {altchars!r}') + if ignorechars is _NOT_SPECIFIED: + for b in b'+/': + if b not in altchars and b in s: + badchar = b + break + s = s.translate(bytes.maketrans(altchars, b'+/')) + else: + alphabet = binascii.BASE64_ALPHABET[:-2] + altchars + return binascii.a2b_base64(s, strict_mode=validate, + alphabet=alphabet, + padded=padded, ignorechars=ignorechars, + canonical=canonical) + if ignorechars is _NOT_SPECIFIED: + ignorechars = b'' + result = binascii.a2b_base64(s, strict_mode=validate, + padded=padded, ignorechars=ignorechars, + canonical=canonical) + if badchar is not None: + import warnings + if validate: + warnings.warn(f'invalid character {chr(badchar)!a} in Base64 data ' + f'with altchars={altchars!r} and validate=True ' + f'will be an error in future Python versions', + DeprecationWarning, stacklevel=2) + else: + warnings.warn(f'invalid character {chr(badchar)!a} in Base64 data ' + f'with altchars={altchars!r} and validate=False ' + f'will be discarded in future Python versions', + FutureWarning, stacklevel=2) + return result def standard_b64encode(s): @@ -103,19 +150,21 @@ def standard_b64decode(s): return b64decode(s) -_urlsafe_encode_translation = bytes.maketrans(b'+/', b'-_') _urlsafe_decode_translation = bytes.maketrans(b'-_', b'+/') -def urlsafe_b64encode(s): +def urlsafe_b64encode(s, *, padded=True): """Encode bytes using the URL- and filesystem-safe Base64 alphabet. Argument s is a bytes-like object to encode. The result is returned as a bytes object. The alphabet uses '-' instead of '+' and '_' instead of '/'. + + If padded is false, omit padding in the output. """ - return b64encode(s).translate(_urlsafe_encode_translation) + return binascii.b2a_base64(s, padded=padded, newline=False, + alphabet=binascii.URLSAFE_BASE64_ALPHABET) -def urlsafe_b64decode(s): +def urlsafe_b64decode(s, *, padded=False): """Decode bytes using the URL- and filesystem-safe Base64 alphabet. Argument s is a bytes-like object or ASCII string to decode. The result @@ -124,30 +173,53 @@ def urlsafe_b64decode(s): alphabet, and are not a plus '+' or slash '/', are discarded prior to the padding check. + If padded is false, padding in input is not required. + The alphabet uses '-' instead of '+' and '_' instead of '/'. """ s = _bytes_from_decode_data(s) + badchar = None + for b in b'+/': + if b in s: + badchar = b + break s = s.translate(_urlsafe_decode_translation) - return b64decode(s) + result = binascii.a2b_base64(s, strict_mode=False, padded=padded) + if badchar is not None: + import warnings + warnings.warn(f'invalid character {chr(badchar)!a} in URL-safe Base64 data ' + f'will be discarded in future Python versions', + FutureWarning, stacklevel=2) + return result # Base32 encoding/decoding must be done in Python _B32_ENCODE_DOCSTRING = ''' Encode the bytes-like objects using {encoding} and return a bytes object. + +If padded is false, omit padding in the output. + +If wrapcol is non-zero, insert a newline (b'\\n') character after at most +every wrapcol characters. ''' _B32_DECODE_DOCSTRING = ''' Decode the {encoding} encoded bytes-like object or ASCII string s. Optional casefold is a flag specifying whether a lowercase alphabet is acceptable as input. For security purposes, the default is False. + +If padded is false, padding in input is not required. + +ignorechars should be a byte string containing characters to ignore +from the input. {extra_args} The result is returned as a bytes object. A binascii.Error is raised if the input is incorrectly padded or if there are non-alphabet characters present in the input. ''' _B32_DECODE_MAP01_DOCSTRING = ''' -RFC 3548 allows for optional mapping of the digit 0 (zero) to the +RFC 4648 allows for optional mapping of the digit 0 (zero) to the letter O (oh), and for optional mapping of the digit 1 (one) to either the letter I (eye) or letter L (el). The optional argument map01 when not None, specifies which letter the digit 1 should be @@ -155,371 +227,183 @@ def urlsafe_b64decode(s): the letter O). For security purposes the default is None, so that 0 and 1 are not allowed in the input. ''' -_b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567' -_b32hexalphabet = b'0123456789ABCDEFGHIJKLMNOPQRSTUV' -_b32tab2 = {} -_b32rev = {} - -def _b32encode(alphabet, s): - # Delay the initialization of the table to not waste memory - # if the function is never called - if alphabet not in _b32tab2: - b32tab = [bytes((i,)) for i in alphabet] - _b32tab2[alphabet] = [a + b for a in b32tab for b in b32tab] - b32tab = None - - if not isinstance(s, bytes_types): - s = memoryview(s).tobytes() - leftover = len(s) % 5 - # Pad the last quantum with zero bits if necessary - if leftover: - s = s + b'\0' * (5 - leftover) # Don't use += ! - encoded = bytearray() - from_bytes = int.from_bytes - b32tab2 = _b32tab2[alphabet] - for i in range(0, len(s), 5): - c = from_bytes(s[i: i + 5]) # big endian - encoded += (b32tab2[c >> 30] + # bits 1 - 10 - b32tab2[(c >> 20) & 0x3ff] + # bits 11 - 20 - b32tab2[(c >> 10) & 0x3ff] + # bits 21 - 30 - b32tab2[c & 0x3ff] # bits 31 - 40 - ) - # Adjust for any leftover partial quanta - if leftover == 1: - encoded[-6:] = b'======' - elif leftover == 2: - encoded[-4:] = b'====' - elif leftover == 3: - encoded[-3:] = b'===' - elif leftover == 4: - encoded[-1:] = b'=' - return bytes(encoded) - -def _b32decode(alphabet, s, casefold=False, map01=None): - # Delay the initialization of the table to not waste memory - # if the function is never called - if alphabet not in _b32rev: - _b32rev[alphabet] = {v: k for k, v in enumerate(alphabet)} + +def b32encode(s, *, padded=True, wrapcol=0): + return binascii.b2a_base32(s, padded=padded, wrapcol=wrapcol) +b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32') + +def b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b'', + canonical=False): s = _bytes_from_decode_data(s) - if len(s) % 8: - raise binascii.Error('Incorrect padding') # Handle section 2.4 zero and one mapping. The flag map01 will be either # False, or the character to map the digit 1 (one) to. It should be # either L (el) or I (eye). if map01 is not None: map01 = _bytes_from_decode_data(map01) - assert len(map01) == 1, repr(map01) s = s.translate(bytes.maketrans(b'01', b'O' + map01)) if casefold: s = s.upper() - # Strip off pad characters from the right. We need to count the pad - # characters because this will tell us how many null bytes to remove from - # the end of the decoded string. - l = len(s) - s = s.rstrip(b'=') - padchars = l - len(s) - # Now decode the full quanta - decoded = bytearray() - b32rev = _b32rev[alphabet] - for i in range(0, len(s), 8): - quanta = s[i: i + 8] - acc = 0 - try: - for c in quanta: - acc = (acc << 5) + b32rev[c] - except KeyError: - raise binascii.Error('Non-base32 digit found') from None - decoded += acc.to_bytes(5) # big endian - # Process the last, partial quanta - if l % 8 or padchars not in {0, 1, 3, 4, 6}: - raise binascii.Error('Incorrect padding') - if padchars and decoded: - acc <<= 5 * padchars - last = acc.to_bytes(5) # big endian - leftover = (43 - 5 * padchars) // 8 # 1: 4, 3: 3, 4: 2, 6: 1 - decoded[-5:] = last[:leftover] - return bytes(decoded) - - -def b32encode(s): - return _b32encode(_b32alphabet, s) -b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32') - -def b32decode(s, casefold=False, map01=None): - return _b32decode(_b32alphabet, s, casefold, map01) + return binascii.a2b_base32(s, padded=padded, ignorechars=ignorechars, + canonical=canonical) b32decode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32', extra_args=_B32_DECODE_MAP01_DOCSTRING) -def b32hexencode(s): - return _b32encode(_b32hexalphabet, s) +def b32hexencode(s, *, padded=True, wrapcol=0): + return binascii.b2a_base32(s, padded=padded, wrapcol=wrapcol, + alphabet=binascii.BASE32HEX_ALPHABET) b32hexencode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32hex') -def b32hexdecode(s, casefold=False): +def b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b'', + canonical=False): + s = _bytes_from_decode_data(s) # base32hex does not have the 01 mapping - return _b32decode(_b32hexalphabet, s, casefold) + if casefold: + s = s.upper() + return binascii.a2b_base32(s, alphabet=binascii.BASE32HEX_ALPHABET, + padded=padded, ignorechars=ignorechars, + canonical=canonical) b32hexdecode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32hex', extra_args='') -# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns +# RFC 4648, Base 16 Alphabet specifies uppercase, but hexlify() returns # lowercase. The RFC also recommends against accepting input case # insensitively. -def b16encode(s): +def b16encode(s, *, wrapcol=0): """Encode the bytes-like object s using Base16 and return a bytes object. + + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. """ - return binascii.hexlify(s).upper() + if not wrapcol: + return binascii.hexlify(s).upper() + if wrapcol < 0: + raise ValueError('Negative wrapcol') + if wrapcol < 2: + wrapcol = 2 + return binascii.hexlify(s, bytes_per_sep=-(wrapcol//2), sep=b'\n').upper() -def b16decode(s, casefold=False): +def b16decode(s, casefold=False, *, ignorechars=b''): """Decode the Base16 encoded bytes-like object or ASCII string s. Optional casefold is a flag specifying whether a lowercase alphabet is acceptable as input. For security purposes, the default is False. + ignorechars should be a byte string containing characters to ignore + from the input. + The result is returned as a bytes object. A binascii.Error is raised if s is incorrectly padded or if there are non-alphabet characters present in the input. """ - s = _bytes_from_decode_data(s) - if casefold: - s = s.upper() - if s.translate(None, delete=b'0123456789ABCDEF'): - raise binascii.Error('Non-base16 digit found') - return binascii.unhexlify(s) + if not casefold: + s = _bytes_from_decode_data(s) + if not isinstance(ignorechars, bytes): + ignorechars = bytes(memoryview(ignorechars)) + for b in b'abcdef': + if b in s and b not in ignorechars: + raise binascii.Error('Non-base16 digit found') + s = s.translate(None, delete=b'abcdef') + return binascii.unhexlify(s, ignorechars=ignorechars) # # Ascii85 encoding/decoding # - -_a85chars = None -_a85chars2 = None -_A85START = b"<~" -_A85END = b"~>" - -def _85encode(b, chars, chars2, pad=False, foldnuls=False, foldspaces=False): - # Helper function for a85encode and b85encode - if not isinstance(b, bytes_types): - b = memoryview(b).tobytes() - - padding = (-len(b)) % 4 - if padding: - b = b + b'\0' * padding - words = struct.Struct('!%dI' % (len(b) // 4)).unpack(b) - - chunks = [b'z' if foldnuls and not word else - b'y' if foldspaces and word == 0x20202020 else - (chars2[word // 614125] + - chars2[word // 85 % 7225] + - chars[word % 85]) - for word in words] - - if padding and not pad: - if chunks[-1] == b'z': - chunks[-1] = chars[0] * 5 - chunks[-1] = chunks[-1][:-padding] - - return b''.join(chunks) - def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): """Encode bytes-like object b using Ascii85 and return a bytes object. foldspaces is an optional flag that uses the special short sequence 'y' instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Adobe encoding. + feature is not supported by the standard encoding used in PDF. - wrapcol controls whether the output should have newline (b'\\n') characters - added to it. If this is non-zero, each output line will be at most this - many characters long, excluding the trailing newline. + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. - pad controls whether the input is padded to a multiple of 4 before - encoding. Note that the btoa implementation always pads. + pad controls whether zero-padding applied to the end of the input + is fully retained in the output encoding, as done by btoa, + producing an exact multiple of 5 bytes of output. - adobe controls whether the encoded byte sequence is framed with <~ and ~>, - which is used by the Adobe implementation. - """ - global _a85chars, _a85chars2 - # Delay the initialization of tables to not waste memory - # if the function is never called - if _a85chars2 is None: - _a85chars = [bytes((i,)) for i in range(33, 118)] - _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars] - - result = _85encode(b, _a85chars, _a85chars2, pad, True, foldspaces) - - if adobe: - result = _A85START + result - if wrapcol: - wrapcol = max(2 if adobe else 1, wrapcol) - chunks = [result[i: i + wrapcol] - for i in range(0, len(result), wrapcol)] - if adobe: - if len(chunks[-1]) + 2 > wrapcol: - chunks.append(b'') - result = b'\n'.join(chunks) - if adobe: - result += _A85END + adobe controls whether the encoded byte sequence is framed with <~ + and ~>, as in a PostScript base-85 string literal. Note that + while ASCII85Decode streams in PDF documents must be terminated + with ~>, they must not use a leading <~. - return result + """ + return binascii.b2a_ascii85(b, foldspaces=foldspaces, + adobe=adobe, wrapcol=wrapcol, pad=pad) -def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'): +def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v', + canonical=False): """Decode the Ascii85 encoded bytes-like object or ASCII string b. - foldspaces is a flag that specifies whether the 'y' short sequence should be - accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is - not supported by the "standard" Adobe encoding. + foldspaces is a flag that specifies whether the 'y' short sequence + should be accepted as shorthand for 4 consecutive spaces (ASCII + 0x20). This feature is not supported by the standard Ascii85 + encoding used in PDF and PostScript. - adobe controls whether the input sequence is in Adobe Ascii85 format (i.e. - is framed with <~ and ~>). + adobe controls whether the <~ and ~> markers are present. While + the leading <~ is not required, the input must end with ~>, or a + ValueError is raised. ignorechars should be a byte string containing characters to ignore from the input. This should only contain whitespace characters, and by default contains all whitespace characters in ASCII. + If canonical is true, non-canonical encodings are rejected. + The result is returned as a bytes object. """ - b = _bytes_from_decode_data(b) - if adobe: - if not b.endswith(_A85END): - raise ValueError( - "Ascii85 encoded byte sequences must end " - "with {!r}".format(_A85END) - ) - if b.startswith(_A85START): - b = b[2:-2] # Strip off start/end markers - else: - b = b[:-2] - # - # We have to go through this stepwise, so as to ignore spaces and handle - # special short sequences - # - packI = struct.Struct('!I').pack - decoded = [] - decoded_append = decoded.append - curr = [] - curr_append = curr.append - curr_clear = curr.clear - for x in b + b'u' * 4: - if b'!'[0] <= x <= b'u'[0]: - curr_append(x) - if len(curr) == 5: - acc = 0 - for x in curr: - acc = 85 * acc + (x - 33) - try: - decoded_append(packI(acc)) - except struct.error: - raise ValueError('Ascii85 overflow') from None - curr_clear() - elif x == b'z'[0]: - if curr: - raise ValueError('z inside Ascii85 5-tuple') - decoded_append(b'\0\0\0\0') - elif foldspaces and x == b'y'[0]: - if curr: - raise ValueError('y inside Ascii85 5-tuple') - decoded_append(b'\x20\x20\x20\x20') - elif x in ignorechars: - # Skip whitespace - continue - else: - raise ValueError('Non-Ascii85 digit found: %c' % x) - - result = b''.join(decoded) - padding = 4 - len(curr) - if padding: - # Throw away the extra padding - result = result[:-padding] - return result - -# The following code is originally taken (with permission) from Mercurial + return binascii.a2b_ascii85(b, foldspaces=foldspaces, + adobe=adobe, ignorechars=ignorechars, + canonical=canonical) -_b85alphabet = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" - b"abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~") -_b85chars = None -_b85chars2 = None -_b85dec = None - -def b85encode(b, pad=False): +def b85encode(b, pad=False, *, wrapcol=0): """Encode bytes-like object b in base85 format and return a bytes object. - If pad is true, the input is padded with b'\\0' so its length is a multiple of - 4 bytes before encoding. + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. + + The input is padded with b'\0' so its length is a multiple of 4 + bytes before encoding. If pad is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes. """ - global _b85chars, _b85chars2 - # Delay the initialization of tables to not waste memory - # if the function is never called - if _b85chars2 is None: - _b85chars = [bytes((i,)) for i in _b85alphabet] - _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars] - return _85encode(b, _b85chars, _b85chars2, pad) - -def b85decode(b): + return binascii.b2a_base85(b, wrapcol=wrapcol, pad=pad) + +def b85decode(b, *, ignorechars=b'', canonical=False): """Decode the base85-encoded bytes-like object or ASCII string b + If canonical is true, non-canonical encodings are rejected. + The result is returned as a bytes object. """ - global _b85dec - # Delay the initialization of tables to not waste memory - # if the function is never called - if _b85dec is None: - _b85dec = [None] * 256 - for i, c in enumerate(_b85alphabet): - _b85dec[c] = i - - b = _bytes_from_decode_data(b) - padding = (-len(b)) % 5 - b = b + b'~' * padding - out = [] - packI = struct.Struct('!I').pack - for i in range(0, len(b), 5): - chunk = b[i:i + 5] - acc = 0 - try: - for c in chunk: - acc = acc * 85 + _b85dec[c] - except TypeError: - for j, c in enumerate(chunk): - if _b85dec[c] is None: - raise ValueError('bad base85 character at position %d' - % (i + j)) from None - raise - try: - out.append(packI(acc)) - except struct.error: - raise ValueError('base85 overflow in hunk starting at byte %d' - % i) from None - - result = b''.join(out) - if padding: - result = result[:-padding] - return result + return binascii.a2b_base85(b, ignorechars=ignorechars, + canonical=canonical) + +def z85encode(s, pad=False, *, wrapcol=0): + """Encode bytes-like object b in z85 format and return a bytes object. -_z85alphabet = (b'0123456789abcdefghijklmnopqrstuvwxyz' - b'ABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#') -# Translating b85 valid but z85 invalid chars to b'\x00' is required -# to prevent them from being decoded as b85 valid chars. -_z85_b85_decode_diff = b';_`|~' -_z85_decode_translation = bytes.maketrans( - _z85alphabet + _z85_b85_decode_diff, - _b85alphabet + b'\x00' * len(_z85_b85_decode_diff) -) -_z85_encode_translation = bytes.maketrans(_b85alphabet, _z85alphabet) - -def z85encode(s): - """Encode bytes-like object b in z85 format and return a bytes object.""" - return b85encode(s).translate(_z85_encode_translation) - -def z85decode(s): + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. + + The input is padded with b'\0' so its length is a multiple of + bytes before encoding. If pad is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, as required by the ZeroMQ standard. + """ + return binascii.b2a_base85(s, wrapcol=wrapcol, pad=pad, + alphabet=binascii.Z85_ALPHABET) + +def z85decode(s, *, ignorechars=b'', canonical=False): """Decode the z85-encoded bytes-like object or ASCII string b + If canonical is true, non-canonical encodings are rejected. + The result is returned as a bytes object. """ - s = _bytes_from_decode_data(s) - s = s.translate(_z85_decode_translation) - try: - return b85decode(s) - except ValueError as e: - raise ValueError(e.args[0].replace('base85', 'z85')) from None + return binascii.a2b_base85(s, alphabet=binascii.Z85_ALPHABET, + ignorechars=ignorechars, canonical=canonical) # Legacy interface. This code could be cleaned up since I don't believe # binascii has any line length limitations. It just doesn't seem worth it @@ -563,11 +447,10 @@ def encodebytes(s): """Encode a bytestring into a bytes object containing multiple lines of base-64 data.""" _input_type_check(s) - pieces = [] - for i in range(0, len(s), MAXBINSIZE): - chunk = s[i : i + MAXBINSIZE] - pieces.append(binascii.b2a_base64(chunk)) - return b"".join(pieces) + result = binascii.b2a_base64(s, wrapcol=MAXLINESIZE) + if result == b'\n': + return b'' + return result def decodebytes(s): @@ -601,7 +484,14 @@ def main(): with open(args[0], 'rb') as f: func(f, sys.stdout.buffer) else: - func(sys.stdin.buffer, sys.stdout.buffer) + if sys.stdin.isatty(): + # gh-138775: read terminal input data all at once to detect EOF + import io + data = sys.stdin.buffer.read() + buffer = io.BytesIO(data) + else: + buffer = sys.stdin.buffer + func(buffer, sys.stdout.buffer) if __name__ == '__main__': diff --git a/Lib/bdb.py b/Lib/bdb.py index efc3e0a235ac8e1..50cf2b3f5b3e453 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -199,6 +199,8 @@ def __init__(self, skip=None, backend='settrace'): self.frame_returning = None self.trace_opcodes = False self.enterframe = None + self.cmdframe = None + self.cmdlineno = None self.code_linenos = weakref.WeakKeyDictionary() self.backend = backend if backend == 'monitoring': @@ -297,7 +299,12 @@ def dispatch_line(self, frame): self.user_line(). Raise BdbQuit if self.quitting is set. Return self.trace_dispatch to continue tracing in this scope. """ - if self.stop_here(frame) or self.break_here(frame): + # GH-136057 + # For line events, we don't want to stop at the same line where + # the latest next/step command was issued. + if (self.stop_here(frame) or self.break_here(frame)) and not ( + self.cmdframe == frame and self.cmdlineno == frame.f_lineno + ): self.user_line(frame) self.restart_events() if self.quitting: raise BdbQuit @@ -526,7 +533,8 @@ def _set_trace_opcodes(self, trace_opcodes): if self.monitoring_tracer: self.monitoring_tracer.update_local_events() - def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): + def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False, + cmdframe=None, cmdlineno=None): """Set the attributes for stopping. If stoplineno is greater than or equal to 0, then stop at line @@ -539,6 +547,10 @@ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): # stoplineno >= 0 means: stop at line >= the stoplineno # stoplineno -1 means: don't stop at all self.stoplineno = stoplineno + # cmdframe/cmdlineno is the frame/line number when the user issued + # step/next commands. + self.cmdframe = cmdframe + self.cmdlineno = cmdlineno self._set_trace_opcodes(opcode) def _set_caller_tracefunc(self, current_frame): @@ -564,7 +576,9 @@ def set_until(self, frame, lineno=None): def set_step(self): """Stop after one line of code.""" - self._set_stopinfo(None, None) + # set_step() could be called from signal handler so enterframe might be None + self._set_stopinfo(None, None, cmdframe=self.enterframe, + cmdlineno=getattr(self.enterframe, 'f_lineno', None)) def set_stepinstr(self): """Stop before the next instruction.""" @@ -572,7 +586,7 @@ def set_stepinstr(self): def set_next(self, frame): """Stop on the next line in or below the given frame.""" - self._set_stopinfo(frame, None) + self._set_stopinfo(frame, None, cmdframe=frame, cmdlineno=frame.f_lineno) def set_return(self, frame): """Stop when returning from the given frame.""" diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 770d26f79628fe1..cc6255f61ae2114 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -1,205 +1,13 @@ -"""Python interface for the 'lsprof' profiler. - Compatible with the 'profile' module. -""" - -__all__ = ["run", "runctx", "Profile"] - -import _lsprof -import importlib.machinery -import importlib.util -import io -import profile as _pyprofile - -# ____________________________________________________________ -# Simple interface - -def run(statement, filename=None, sort=-1): - return _pyprofile._Utils(Profile).run(statement, filename, sort) - -def runctx(statement, globals, locals, filename=None, sort=-1): - return _pyprofile._Utils(Profile).runctx(statement, globals, locals, - filename, sort) - -run.__doc__ = _pyprofile.run.__doc__ -runctx.__doc__ = _pyprofile.runctx.__doc__ - -# ____________________________________________________________ - -class Profile(_lsprof.Profiler): - """Profile(timer=None, timeunit=None, subcalls=True, builtins=True) - - Builds a profiler object using the specified timer function. - The default timer is a fast built-in one based on real time. - For custom timer functions returning integers, timeunit can - be a float specifying a scale (i.e. how long each integer unit - is, in seconds). - """ - - # Most of the functionality is in the base class. - # This subclass only adds convenient and backward-compatible methods. - - def print_stats(self, sort=-1): - import pstats - if not isinstance(sort, tuple): - sort = (sort,) - pstats.Stats(self).strip_dirs().sort_stats(*sort).print_stats() - - def dump_stats(self, file): - import marshal - with open(file, 'wb') as f: - self.create_stats() - marshal.dump(self.stats, f) - - def create_stats(self): - self.disable() - self.snapshot_stats() +"""Compatibility wrapper for cProfile module. - def snapshot_stats(self): - entries = self.getstats() - self.stats = {} - callersdicts = {} - # call information - for entry in entries: - func = label(entry.code) - nc = entry.callcount # ncalls column of pstats (before '/') - cc = nc - entry.reccallcount # ncalls column of pstats (after '/') - tt = entry.inlinetime # tottime column of pstats - ct = entry.totaltime # cumtime column of pstats - callers = {} - callersdicts[id(entry.code)] = callers - self.stats[func] = cc, nc, tt, ct, callers - # subcall information - for entry in entries: - if entry.calls: - func = label(entry.code) - for subentry in entry.calls: - try: - callers = callersdicts[id(subentry.code)] - except KeyError: - continue - nc = subentry.callcount - cc = nc - subentry.reccallcount - tt = subentry.inlinetime - ct = subentry.totaltime - if func in callers: - prev = callers[func] - nc += prev[0] - cc += prev[1] - tt += prev[2] - ct += prev[3] - callers[func] = nc, cc, tt, ct - - # The following two methods can be called by clients to use - # a profiler to profile a statement, given as a string. - - def run(self, cmd): - import __main__ - dict = __main__.__dict__ - return self.runctx(cmd, dict, dict) - - def runctx(self, cmd, globals, locals): - self.enable() - try: - exec(cmd, globals, locals) - finally: - self.disable() - return self - - # This method is more useful to profile a single function call. - def runcall(self, func, /, *args, **kw): - self.enable() - try: - return func(*args, **kw) - finally: - self.disable() - - def __enter__(self): - self.enable() - return self - - def __exit__(self, *exc_info): - self.disable() - -# ____________________________________________________________ - -def label(code): - if isinstance(code, str): - return ('~', 0, code) # built-in functions ('~' sorts at the end) - else: - return (code.co_filename, code.co_firstlineno, code.co_name) - -# ____________________________________________________________ - -def main(): - import os - import sys - import runpy - import pstats - from optparse import OptionParser - usage = "cProfile.py [-o output_file_path] [-s sort] [-m module | scriptfile] [arg] ..." - parser = OptionParser(usage=usage) - parser.allow_interspersed_args = False - parser.add_option('-o', '--outfile', dest="outfile", - help="Save stats to ", default=None) - parser.add_option('-s', '--sort', dest="sort", - help="Sort order when printing to stdout, based on pstats.Stats class", - default=2, - choices=sorted(pstats.Stats.sort_arg_dict_default)) - parser.add_option('-m', dest="module", action="store_true", - help="Profile a library module", default=False) - - if not sys.argv[1:]: - parser.print_usage() - sys.exit(2) - - (options, args) = parser.parse_args() - sys.argv[:] = args - - # The script that we're profiling may chdir, so capture the absolute path - # to the output file at startup. - if options.outfile is not None: - options.outfile = os.path.abspath(options.outfile) +This module maintains backward compatibility by importing from the new +profiling.tracing module. +""" - if len(args) > 0: - if options.module: - code = "run_module(modname, run_name='__main__')" - globs = { - 'run_module': runpy.run_module, - 'modname': args[0] - } - else: - progname = args[0] - sys.path.insert(0, os.path.dirname(progname)) - with io.open_code(progname) as fp: - code = compile(fp.read(), progname, 'exec') - spec = importlib.machinery.ModuleSpec(name='__main__', loader=None, - origin=progname) - module = importlib.util.module_from_spec(spec) - # Set __main__ so that importing __main__ in the profiled code will - # return the same namespace that the code is executing under. - sys.modules['__main__'] = module - # Ensure that we're using the same __dict__ instance as the module - # for the global variables so that updates to globals are reflected - # in the module's namespace. - globs = module.__dict__ - globs.update({ - '__spec__': spec, - '__file__': spec.origin, - '__name__': spec.name, - '__package__': None, - '__cached__': None, - }) +from profiling.tracing import run, runctx, Profile - try: - runctx(code, globs, None, options.outfile, options.sort) - except BrokenPipeError as exc: - # Prevent "Exception ignored" during interpreter shutdown. - sys.stdout = None - sys.exit(exc.errno) - else: - parser.print_usage() - return parser +__all__ = ["run", "runctx", "Profile"] -# When invoked as main program, invoke the profiler on a script -if __name__ == '__main__': +if __name__ == "__main__": + from profiling.tracing.__main__ import main main() diff --git a/Lib/calendar.py b/Lib/calendar.py index 3be1b50500eb070..92fe6b7723fe268 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -14,8 +14,9 @@ __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday", "firstweekday", "isleap", "leapdays", "weekday", "monthrange", "monthcalendar", "prmonth", "month", "prcal", "calendar", - "timegm", "month_name", "month_abbr", "day_name", "day_abbr", - "Calendar", "TextCalendar", "HTMLCalendar", "LocaleTextCalendar", + "timegm", "month_name", "month_abbr", "standalone_month_name", + "standalone_month_abbr", "day_name", "day_abbr", "Calendar", + "TextCalendar", "HTMLCalendar", "LocaleTextCalendar", "LocaleHTMLCalendar", "weekheader", "Day", "Month", "JANUARY", "FEBRUARY", "MARCH", "APRIL", "MAY", "JUNE", "JULY", @@ -139,6 +140,24 @@ def __len__(self): month_name = _localized_month('%B') month_abbr = _localized_month('%b') +# On platforms that support the %OB and %Ob specifiers, they are used +# to get the standalone form of the month name. This is required for +# some languages such as Greek, Slavic, and Baltic languages. +try: + standalone_month_name = _localized_month('%OB') + standalone_month_abbr = _localized_month('%Ob') +except ValueError: + standalone_month_name = month_name + standalone_month_abbr = month_abbr +else: + # Some systems that do not support '%OB' will keep it as-is (i.e., + # we get [..., '%OB', '%OB', '%OB']), so for non-distinct names, + # we fall back to month_name/month_abbr. + if len(set(standalone_month_name)) != len(set(month_name)): + standalone_month_name = month_name + if len(set(standalone_month_abbr)) != len(set(month_abbr)): + standalone_month_abbr = month_abbr + def isleap(year): """Return True for leap years, False for non-leap years.""" @@ -359,7 +378,7 @@ def formatweekday(self, day, width): """ Returns a formatted week day name. """ - if width >= 9: + if width >= max(map(len, day_name)): names = day_name else: names = day_abbr @@ -377,7 +396,7 @@ def formatmonthname(self, theyear, themonth, width, withyear=True): """ _validate_month(themonth) - s = month_name[themonth] + s = standalone_month_name[themonth] if withyear: s = "%s %r" % (s, theyear) return s.center(width) @@ -479,30 +498,29 @@ def formatday(self, day, weekday): """ if day == 0: # day outside month - return ' ' % self.cssclass_noday + return f' ' else: - return '%d' % (self.cssclasses[weekday], day) + return f'{day}' def formatweek(self, theweek): """ Return a complete week as a table row. """ s = ''.join(self.formatday(d, wd) for (d, wd) in theweek) - return '%s' % s + return f'{s}' def formatweekday(self, day): """ Return a weekday name as a table header. """ - return '%s' % ( - self.cssclasses_weekday_head[day], day_abbr[day]) + return f'{day_abbr[day]}' def formatweekheader(self): """ Return a header for a week as a table row. """ s = ''.join(self.formatweekday(i) for i in self.iterweekdays()) - return '%s' % s + return f'{s}' def formatmonthname(self, theyear, themonth, withyear=True): """ @@ -510,11 +528,10 @@ def formatmonthname(self, theyear, themonth, withyear=True): """ _validate_month(themonth) if withyear: - s = '%s %s' % (month_name[themonth], theyear) + s = f'{standalone_month_name[themonth]} {theyear}' else: - s = '%s' % month_name[themonth] - return '%s' % ( - self.cssclass_month_head, s) + s = standalone_month_name[themonth] + return f'{s}' def formatmonth(self, theyear, themonth, withyear=True): """ @@ -522,8 +539,7 @@ def formatmonth(self, theyear, themonth, withyear=True): """ v = [] a = v.append - a('' % ( - self.cssclass_month)) + a(f'
') a('\n') a(self.formatmonthname(theyear, themonth, withyear=withyear)) a('\n') @@ -543,11 +559,9 @@ def formatyear(self, theyear, width=3): v = [] a = v.append width = max(width, 1) - a('
' % - self.cssclass_year) + a(f'
') a('\n') - a('' % ( - width, self.cssclass_year_head, theyear)) + a(f'') for i in range(JANUARY, JANUARY+12, width): # months in this row months = range(i, min(i+width, 13)) @@ -560,29 +574,48 @@ def formatyear(self, theyear, width=3): a('
%s
{theyear}
') return ''.join(v) - def formatyearpage(self, theyear, width=3, css='calendar.css', encoding=None): + def _format_html_page(self, theyear, content, css, encoding): """ - Return a formatted year as a complete HTML page. + Return a complete HTML page with the given content. """ if encoding is None: encoding = 'utf-8' v = [] a = v.append - a('\n' % encoding) - a('\n') - a('\n') + a('\n') + a('\n') a('\n') - a('\n' % encoding) + a(f'\n') + a('\n') + a(f'Calendar for {theyear}\n') + a('\n') if css is not None: - a('\n' % css) - a('Calendar for %d\n' % theyear) + a(f'\n') a('\n') a('\n') - a(self.formatyear(theyear, width)) + a(content) a('\n') a('\n') return ''.join(v).encode(encoding, "xmlcharrefreplace") + def formatyearpage(self, theyear, width=3, css='calendar.css', encoding=None): + """ + Return a formatted year as a complete HTML page. + """ + content = self.formatyear(theyear, width) + return self._format_html_page(theyear, content, css, encoding) + + def formatmonthpage(self, theyear, themonth, width=3, css='calendar.css', encoding=None): + """ + Return a formatted month as a complete HTML page. + """ + content = self.formatmonth(theyear, themonth, width) + return self._format_html_page(theyear, content, css, encoding) + class different_locale: def __init__(self, locale): @@ -653,28 +686,61 @@ def __init__(self, highlight_day=None, *args, **kwargs): super().__init__(*args, **kwargs) self.highlight_day = highlight_day - def formatweek(self, theweek, width, *, highlight_day=None): + def _get_theme(self): + from _colorize import get_theme + + return get_theme(tty_file=sys.stdout) + + def formatday(self, day, weekday, width, *, highlight_day=None): """ - Returns a single week in a string (no newline). + Returns a formatted day. """ - if highlight_day: - from _colorize import get_colors - - ansi = get_colors() - highlight = f"{ansi.BLACK}{ansi.BACKGROUND_YELLOW}" - reset = ansi.RESET + if day == 0: + s = '' else: - highlight = reset = "" + s = f'{day:2}' + s = s.center(width) + if day == highlight_day: + theme = self._get_theme().calendar + s = f"{theme.highlight}{s}{theme.reset}" + return s + def formatweek(self, theweek, width, *, highlight_day=None): + """ + Returns a single week in a string (no newline). + """ return ' '.join( - ( - f"{highlight}{self.formatday(d, wd, width)}{reset}" - if d == highlight_day - else self.formatday(d, wd, width) - ) + self.formatday(d, wd, width, highlight_day=highlight_day) for (d, wd) in theweek ) + def formatweekheader(self, width): + """ + Return a header for a week. + """ + header = super().formatweekheader(width) + theme = self._get_theme().calendar + return f"{theme.weekday}{header}{theme.reset}" + + def formatmonthname(self, theyear, themonth, width, withyear=True): + """ + Return a formatted month name. + """ + name = super().formatmonthname(theyear, themonth, width, withyear) + theme = self._get_theme().calendar + if ( + self.highlight_day + and self.highlight_day.year == theyear + and self.highlight_day.month == themonth + ): + color = theme.highlight + name_only = name.strip() + colored_name = f"{color}{name_only}{theme.reset}" + return name.replace(name_only, colored_name, 1) + else: + color = theme.header + return f"{color}{name}{theme.reset}" + def formatmonth(self, theyear, themonth, w=0, l=0): """ Return a month's calendar string (multi-line). @@ -709,7 +775,9 @@ def formatyear(self, theyear, w=2, l=1, c=6, m=3): colwidth = (w + 1) * 7 - 1 v = [] a = v.append - a(repr(theyear).center(colwidth*m+c*(m-1)).rstrip()) + theme = self._get_theme().calendar + year = repr(theyear).center(colwidth*m+c*(m-1)).rstrip() + a(f"{theme.header}{year}{theme.reset}") a('\n'*l) header = self.formatweekheader(w) for (i, row) in enumerate(self.yeardays2calendar(theyear, m)): @@ -810,28 +878,30 @@ def timegm(tuple): def main(args=None): import argparse - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) textgroup = parser.add_argument_group('text only arguments') htmlgroup = parser.add_argument_group('html only arguments') textgroup.add_argument( "-w", "--width", type=int, default=2, - help="width of date column (default 2)" + help="width of date column" ) textgroup.add_argument( "-l", "--lines", type=int, default=1, - help="number of lines for each week (default 1)" + help="number of lines for each week" ) textgroup.add_argument( "-s", "--spacing", type=int, default=6, - help="spacing between months (default 6)" + help="spacing between months" ) textgroup.add_argument( "-m", "--months", type=int, default=3, - help="months per row (default 3)" + help="months per row" ) htmlgroup.add_argument( "-c", "--css", @@ -846,18 +916,18 @@ def main(args=None): parser.add_argument( "-e", "--encoding", default=None, - help="encoding to use for output (default utf-8)" + help="encoding to use for output" ) parser.add_argument( "-t", "--type", default="text", choices=("text", "html"), - help="output type (text or html)" + help="output type (`text` or `html`)" ) parser.add_argument( "-f", "--first-weekday", type=int, default=0, - help="weekday (0 is Monday, 6 is Sunday) to start each week (default 0)" + help="weekday (0 is Monday, 6 is Sunday) to start each week" ) parser.add_argument( "year", @@ -867,7 +937,7 @@ def main(args=None): parser.add_argument( "month", nargs='?', type=int, - help="month number (1-12, text only)" + help="month number (1-12)" ) options = parser.parse_args(args) @@ -880,9 +950,6 @@ def main(args=None): today = datetime.date.today() if options.type == "html": - if options.month: - parser.error("incorrect number of arguments") - sys.exit(1) if options.locale: cal = LocaleHTMLCalendar(locale=locale) else: @@ -893,10 +960,14 @@ def main(args=None): encoding = 'utf-8' optdict = dict(encoding=encoding, css=options.css) write = sys.stdout.buffer.write + if options.year is None: write(cal.formatyearpage(today.year, **optdict)) else: - write(cal.formatyearpage(options.year, **optdict)) + if options.month: + write(cal.formatmonthpage(options.year, options.month, **optdict)) + else: + write(cal.formatyearpage(options.year, **optdict)) else: if options.locale: cal = _CLIDemoLocaleCalendar(highlight_day=today, locale=locale) diff --git a/Lib/code.py b/Lib/code.py index f7e275d8801b7c3..df1d7199e339341 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -385,7 +385,7 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=Fa if __name__ == "__main__": import argparse - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument('-q', action='store_true', help="don't print version and copyright messages") args = parser.parse_args() diff --git a/Lib/codecs.py b/Lib/codecs.py index e4a8010aba90a50..af6ab031157e790 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -93,7 +93,7 @@ class CodecInfo(tuple): def __new__(cls, encode, decode, streamreader=None, streamwriter=None, incrementalencoder=None, incrementaldecoder=None, name=None, - *, _is_text_encoding=None): + *, _is_text_encoding=None, _expat_decoding_table=None): self = tuple.__new__(cls, (encode, decode, streamreader, streamwriter)) self.name = name self.encode = encode @@ -104,6 +104,8 @@ def __new__(cls, encode, decode, streamreader=None, streamwriter=None, self.streamreader = streamreader if _is_text_encoding is not None: self._is_text_encoding = _is_text_encoding + if _expat_decoding_table is not None: + self._expat_decoding_table = _expat_decoding_table return self def __repr__(self): diff --git a/Lib/codeop.py b/Lib/codeop.py index 8cac00442d99e3a..40e88423119bc4f 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -66,9 +66,9 @@ def _maybe_compile(compiler, source, filename, symbol, flags): try: compiler(source + "\n", filename, symbol, flags=flags) return None - except _IncompleteInputError as e: + except _IncompleteInputError: return None - except SyntaxError as e: + except SyntaxError: pass # fallthrough diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index b8653f40a942f03..20f1e728733fec6 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -32,6 +32,8 @@ _sys.modules['collections.abc'] = _collections_abc abc = _collections_abc +lazy from copy import copy as _copy +lazy from heapq import nlargest as _nlargest from itertools import chain as _chain from itertools import repeat as _repeat from itertools import starmap as _starmap @@ -59,8 +61,6 @@ except ImportError: pass -heapq = None # Lazily imported - ################################################################################ ### OrderedDict @@ -328,14 +328,14 @@ def __ior__(self, other): return self def __or__(self, other): - if not isinstance(other, dict): + if not isinstance(other, (dict, frozendict)): return NotImplemented new = self.__class__(self) new.update(other) return new def __ror__(self, other): - if not isinstance(other, dict): + if not isinstance(other, (dict, frozendict)): return NotImplemented new = self.__class__(other) new.update(self) @@ -634,12 +634,7 @@ def most_common(self, n=None): if n is None: return sorted(self.items(), key=_itemgetter(1), reverse=True) - # Lazy import to speedup Python startup time - global heapq - if heapq is None: - import heapq - - return heapq.nlargest(n, self.items(), key=_itemgetter(1)) + return _nlargest(n, self.items(), key=_itemgetter(1)) def elements(self): '''Iterator over elements repeating each as many times as its count. @@ -796,6 +791,7 @@ def __repr__(self): # set(cp - cq) == sp - sq # set(cp | cq) == sp | sq # set(cp & cq) == sp & sq + # set(cp ^ cq) == sp ^ sq def __eq__(self, other): 'True if all counts agree. Missing counts are treated as zero.' @@ -908,6 +904,33 @@ def __and__(self, other): result[elem] = newcount return result + def __xor__(self, other): + '''Symmetric difference. Absolute value of count differences. + + The symmetric difference p ^ q is equivalent to: + + (p - q) | (q - p). + + For each element, symmetric difference gives the same result as: + + max(p[elem], q[elem]) - min(p[elem], q[elem]) + + >>> Counter(a=5, b=3, c=2, d=2) ^ Counter(a=1, b=3, c=5, e=1) + Counter({'a': 4, 'c': 3, 'd': 2, 'e': 1}) + + ''' + if not isinstance(other, Counter): + return NotImplemented + result = Counter() + for elem, count in self.items(): + newcount = abs(count - other[elem]) + if newcount: + result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count: + result[elem] = abs(count) + return result + def __pos__(self): 'Adds an empty counter, effectively stripping negative and zero counts' result = Counter() @@ -990,6 +1013,22 @@ def __iand__(self, other): self[elem] = other_count return self._keep_positive() + def __ixor__(self, other): + '''Inplace symmetric difference. Absolute value of count differences. + + >>> c = Counter(a=5, b=3, c=2, d=2) + >>> c ^= Counter(a=1, b=3, c=5, e=1) + >>> c + Counter({'a': 4, 'c': 3, 'd': 2, 'e': 1}) + + ''' + for elem, count in self.items(): + self[elem] = abs(count - other[elem]) + for elem, count in other.items(): + if elem not in self: + self[elem] = abs(count) + return self._keep_positive() + ######################################################################## ### ChainMap @@ -1177,14 +1216,14 @@ def __repr__(self): def __or__(self, other): if isinstance(other, UserDict): return self.__class__(self.data | other.data) - if isinstance(other, dict): + if isinstance(other, (dict, frozendict)): return self.__class__(self.data | other) return NotImplemented def __ror__(self, other): if isinstance(other, UserDict): return self.__class__(other.data | self.data) - if isinstance(other, dict): + if isinstance(other, (dict, frozendict)): return self.__class__(other | self.data) return NotImplemented @@ -1205,16 +1244,29 @@ def __copy__(self): def copy(self): if self.__class__ is UserDict: return UserDict(self.data.copy()) - import copy data = self.data try: self.data = {} - c = copy.copy(self) + c = _copy(self) finally: self.data = data c.update(self) return c + + # This method has a default implementation in MutableMapping, but dict's + # equivalent is last-in, first-out instead of first-in, first-out. + def popitem(self): + """Remove and return a (key, value) pair as a 2-tuple. + + Removes pairs in the same order as the wrapped mapping's popitem() + method. For dict objects (the default), that order is last-in, + first-out (LIFO). + Raises KeyError if the UserDict is empty. + """ + return self.data.popitem() + + @classmethod def fromkeys(cls, iterable, value=None): d = cls() @@ -1498,6 +1550,8 @@ def format_map(self, mapping): return self.data.format_map(mapping) def index(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data return self.data.index(sub, start, end) def isalpha(self): @@ -1566,6 +1620,8 @@ def rfind(self, sub, start=0, end=_sys.maxsize): return self.data.rfind(sub, start, end) def rindex(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data return self.data.rindex(sub, start, end) def rjust(self, width, *args): diff --git a/Lib/compileall.py b/Lib/compileall.py index 67fe370451e1efb..812a496611e043f 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -165,6 +165,14 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, stripdir = os.fspath(stripdir) if stripdir is not None else None name = os.path.basename(fullname) + # Without a cache_tag, we can only create legacy .pyc files. None of our + # callers seem to expect this, so the best we can do is fail without raising + if not legacy and sys.implementation.cache_tag is None: + if not quiet: + print("No cache tag is available to generate .pyc path for", + repr(fullname)) + return False + dfile = None if ddir is not None: @@ -223,7 +231,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, cfile = importlib.util.cache_from_source(fullname) opt_cfiles[opt_level] = cfile - head, tail = name[:-3], name[-3:] + tail = name[-3:] if tail == '.py': if not force: try: @@ -318,7 +326,6 @@ def main(): parser = argparse.ArgumentParser( description='Utilities to support installing Python libraries.', - color=True, ) parser.add_argument('-l', action='store_const', const=0, default=None, dest='maxlevels', @@ -330,10 +337,10 @@ def main(): parser.add_argument('-f', action='store_true', dest='force', help='force rebuild even if timestamps are up to date') parser.add_argument('-q', action='count', dest='quiet', default=0, - help='output only error messages; -qq will suppress ' + help='output only error messages; `-qq` will suppress ' 'the error messages as well.') parser.add_argument('-b', action='store_true', dest='legacy', - help='use legacy (pre-PEP3147) compiled file locations') + help='use legacy (pre-PEP 3147) compiled file locations') parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None, help=('directory to prepend to file paths for use in ' 'compile-time tracebacks and in runtime ' @@ -359,28 +366,28 @@ def main(): 'of each file considered for compilation')) parser.add_argument('-i', metavar='FILE', dest='flist', help=('add all the files and directories listed in ' - 'FILE to the list considered for compilation; ' - 'if "-", names are read from stdin')) + '`FILE` to the list considered for compilation; ' + 'if `"-"`, names are read from `stdin`')) parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*', help=('zero or more file and directory names ' 'to compile; if no arguments given, defaults ' - 'to the equivalent of -l sys.path')) + 'to the equivalent of `-l` `sys.path`')) parser.add_argument('-j', '--workers', default=1, type=int, help='Run compileall concurrently') invalidation_modes = [mode.name.lower().replace('_', '-') for mode in py_compile.PycInvalidationMode] parser.add_argument('--invalidation-mode', choices=sorted(invalidation_modes), - help=('set .pyc invalidation mode; defaults to ' - '"checked-hash" if the SOURCE_DATE_EPOCH ' + help=('set `.pyc` invalidation mode; defaults to ' + '`"checked-hash"` if the `SOURCE_DATE_EPOCH` ' 'environment variable is set, and ' - '"timestamp" otherwise.')) + '`"timestamp"` otherwise.')) parser.add_argument('-o', action='append', type=int, dest='opt_levels', help=('Optimization levels to run compilation with. ' - 'Default is -1 which uses the optimization level ' - 'of the Python interpreter itself (see -O).')) + 'Default is `-1` which uses the optimization level ' + 'of the Python interpreter itself (see `-O`).')) parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', - help='Ignore symlinks pointing outsite of the DIR') + help='Ignore symlinks pointing outsite of the `DIR`') parser.add_argument('--hardlink-dupes', action='store_true', dest='hardlink_dupes', help='Hardlink duplicated pyc files') diff --git a/Lib/compression/zstd/__init__.py b/Lib/compression/zstd/__init__.py index 84b25914b0aa93a..5326cf9b19cf883 100644 --- a/Lib/compression/zstd/__init__.py +++ b/Lib/compression/zstd/__init__.py @@ -61,8 +61,9 @@ def __setattr__(self, name, _): def get_frame_info(frame_buffer): """Get Zstandard frame information from a frame header. - *frame_buffer* is a bytes-like object. It should start from the beginning - of a frame, and needs to include at least the frame header (6 to 18 bytes). + *frame_buffer* is a bytes-like object. It should start from the + beginning of a frame, and needs to include at least the frame header + (6 to 18 bytes). The returned FrameInfo object has two attributes. 'decompressed_size' is the size in bytes of the data in the frame when @@ -103,16 +104,17 @@ def finalize_dict(zstd_dict, /, samples, dict_size, level): finalize *zstd_dict* by adding headers and statistics according to the Zstandard dictionary format. - You may compose an effective dictionary content by hand, which is used as - basis dictionary, and use some samples to finalize a dictionary. The basis - dictionary may be a "raw content" dictionary. See *is_raw* in ZstdDict. + You may compose an effective dictionary content by hand, which is used + as basis dictionary, and use some samples to finalize a dictionary. The + basis dictionary may be a "raw content" dictionary. See *is_raw* in + ZstdDict. - *samples* is an iterable of samples, where a sample is a bytes-like object - representing a file. + *samples* is an iterable of samples, where a sample is a bytes-like + object representing a file. *dict_size* is the dictionary's maximum size, in bytes. *level* is the expected compression level. The statistics for each - compression level differ, so tuning the dictionary to the compression level - can provide improvements. + compression level differ, so tuning the dictionary to the compression + level can provide improvements. """ if not isinstance(zstd_dict, ZstdDict): @@ -140,8 +142,8 @@ def compress(data, level=None, options=None, zstd_dict=None): COMPRESSION_LEVEL_DEFAULT ('3'). *options* is a dict object that contains advanced compression parameters. See CompressionParameter for more on options. - *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See - the function train_dict for how to train a ZstdDict on sample data. + *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. + See the function train_dict for how to train a ZstdDict on sample data. For incremental compression, use a ZstdCompressor instead. """ @@ -152,8 +154,8 @@ def compress(data, level=None, options=None, zstd_dict=None): def decompress(data, zstd_dict=None, options=None): """Decompress one or more frames of Zstandard compressed *data*. - *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See - the function train_dict for how to train a ZstdDict on sample data. + *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. + See the function train_dict for how to train a ZstdDict on sample data. *options* is a dict object that contains advanced compression parameters. See DecompressionParameter for more on options. diff --git a/Lib/compression/zstd/_zstdfile.py b/Lib/compression/zstd/_zstdfile.py index d709f5efc658fa4..8d3358152e9a889 100644 --- a/Lib/compression/zstd/_zstdfile.py +++ b/Lib/compression/zstd/_zstdfile.py @@ -36,9 +36,9 @@ def __init__(self, file, /, mode='r', *, *file* can be either an file-like object, or a file name to open. - *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x' for - creating exclusively, or 'a' for appending. These can equivalently be - given as 'rb', 'wb', 'xb' and 'ab' respectively. + *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x' + for creating exclusively, or 'a' for appending. These can + equivalently be given as 'rb', 'wb', 'xb' and 'ab' respectively. *level* is an optional int specifying the compression level to use, or COMPRESSION_LEVEL_DEFAULT if not given. @@ -296,26 +296,27 @@ def open(file, /, mode='rb', *, level=None, options=None, zstd_dict=None, encoding=None, errors=None, newline=None): """Open a Zstandard compressed file in binary or text mode. - file can be either a file name (given as a str, bytes, or PathLike object), - in which case the named file is opened, or it can be an existing file object - to read from or write to. + file can be either a file name (given as a str, bytes, or PathLike + object), in which case the named file is opened, or it can be + an existing file object to read from or write to. - The mode parameter can be 'r', 'rb' (default), 'w', 'wb', 'x', 'xb', 'a', - 'ab' for binary mode, or 'rt', 'wt', 'xt', 'at' for text mode. + The mode parameter can be 'r', 'rb' (default), 'w', 'wb', 'x', 'xb', + 'a', 'ab' for binary mode, or 'rt', 'wt', 'xt', 'at' for text mode. - The level, options, and zstd_dict parameters specify the settings the same - as ZstdFile. + The level, options, and zstd_dict parameters specify the settings the + same as ZstdFile. When using read mode (decompression), the options parameter is a dict - representing advanced decompression options. The level parameter is not - supported in this case. When using write mode (compression), only one of - level, an int representing the compression level, or options, a dict - representing advanced compression options, may be passed. In both modes, - zstd_dict is a ZstdDict instance containing a trained Zstandard dictionary. - - For binary mode, this function is equivalent to the ZstdFile constructor: - ZstdFile(filename, mode, ...). In this case, the encoding, errors and - newline parameters must not be provided. + representing advanced decompression options. The level parameter is not + supported in this case. When using write mode (compression), only one + of level, an int representing the compression level, or options, a dict + representing advanced compression options, may be passed. In both + modes, zstd_dict is a ZstdDict instance containing a trained Zstandard + dictionary. + + For binary mode, this function is equivalent to the ZstdFile + constructor: ZstdFile(filename, mode, ...). In this case, the encoding, + errors and newline parameters must not be provided. For text mode, an ZstdFile object is created, and wrapped in an io.TextIOWrapper instance with the specified encoding, error handling diff --git a/Lib/concurrent/futures/__init__.py b/Lib/concurrent/futures/__init__.py index e717222cf98b324..d6ac4b3e0b675f6 100644 --- a/Lib/concurrent/futures/__init__.py +++ b/Lib/concurrent/futures/__init__.py @@ -44,7 +44,7 @@ def __dir__(): - return __all__ + ('__author__', '__doc__') + return __all__ + ['__author__', '__doc__'] def __getattr__(name): diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index f506ce68aea5b29..4e71331f9a0a594 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -194,15 +194,15 @@ def as_completed(fs, timeout=None): """An iterator over the given futures that yields each as it completes. Args: - fs: The sequence of Futures (possibly created by different Executors) to - iterate over. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. + fs: The sequence of Futures (possibly created by different + Executors) to iterate over. + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. Returns: - An iterator that yields the given Futures as they complete (finished or - cancelled). If any given Futures are duplicated, they will be returned - once. + An iterator that yields the given Futures as they complete + (finished or cancelled). If any given Futures are duplicated, + they will be returned once. Raises: TimeoutError: If the entire result iterator could not be generated @@ -258,19 +258,20 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED): """Wait for the futures in the given sequence to complete. Args: - fs: The sequence of Futures (possibly created by different Executors) to - wait upon. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. - return_when: Indicates when this function should return. The options - are: + fs: The sequence of Futures (possibly created by different + Executors) to wait upon. + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. + return_when: Indicates when this function should return. + The options are: FIRST_COMPLETED - Return when any future finishes or is cancelled. FIRST_EXCEPTION - Return when any future finishes by raising an - exception. If no future raises an exception + exception. If no future raises an exception then it is equivalent to ALL_COMPLETED. - ALL_COMPLETED - Return when all futures finish or are cancelled. + ALL_COMPLETED - Return when all futures finish or are + cancelled. Returns: A named 2-tuple of sets. The first set, named 'done', contains the @@ -404,11 +405,12 @@ def add_done_callback(self, fn): Args: fn: A callable that will be called with this future as its only - argument when the future completes or is cancelled. The callable - will always be called by a thread in the same process in which - it was added. If the future has already completed or been - cancelled then the callable will be called immediately. These - callables are called in the order that they were added. + argument when the future completes or is cancelled. The + callable will always be called by a thread in the same + process in which it was added. If the future has already + completed or been cancelled then the callable will be + called immediately. These callables are called in the + order that they were added. """ with self._condition: if self._state not in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]: @@ -423,17 +425,19 @@ def result(self, timeout=None): """Return the result of the call that the future represents. Args: - timeout: The number of seconds to wait for the result if the future - isn't done. If None, then there is no limit on the wait time. + timeout: The number of seconds to wait for the result if the + future isn't done. If None, then there is no limit on the + wait time. Returns: The result of the call that the future represents. Raises: CancelledError: If the future was cancelled. - TimeoutError: If the future didn't finish executing before the given - timeout. - Exception: If the call raised then that exception will be raised. + TimeoutError: If the future didn't finish executing before the + given timeout. + Exception: If the call raised then that exception will be + raised. """ try: with self._condition: @@ -459,17 +463,17 @@ def exception(self, timeout=None): Args: timeout: The number of seconds to wait for the exception if the - future isn't done. If None, then there is no limit on the wait - time. + future isn't done. If None, then there is no limit on the + wait time. Returns: - The exception raised by the call that the future represents or None - if the call completed without raising. + The exception raised by the call that the future represents or + None if the call completed without raising. Raises: CancelledError: If the future was cancelled. - TimeoutError: If the future didn't finish executing before the given - timeout. + TimeoutError: If the future didn't finish executing before the + given timeout. """ with self._condition: @@ -494,22 +498,23 @@ def set_running_or_notify_cancel(self): Should only be used by Executor implementations and unit tests. If the future has been cancelled (cancel() was called and returned - True) then any threads waiting on the future completing (though calls - to as_completed() or wait()) are notified and False is returned. + True) then any threads waiting on the future completing (though + calls to as_completed() or wait()) are notified and False is + returned. If the future was not cancelled then it is put in the running state (future calls to running() will return True) and True is returned. This method should be called by Executor implementations before - executing the work associated with this future. If this method returns - False then the work should not be executed. + executing the work associated with this future. If this method + returns False then the work should not be executed. Returns: False if the Future was cancelled, True otherwise. Raises: - RuntimeError: if this method was already called or if set_result() - or set_exception() was called. + RuntimeError: if this method was already called or if + set_result() or set_exception() was called. """ with self._condition: if self._state == CANCELLED: @@ -593,8 +598,9 @@ class Executor(object): def submit(self, fn, /, *args, **kwargs): """Submits a callable to be executed with the given arguments. - Schedules the callable to be executed as fn(*args, **kwargs) and returns - a Future instance representing the execution of the callable. + Schedules the callable to be executed as fn(*args, **kwargs) and + returns a Future instance representing the execution of the + callable. Returns: A Future representing the given call. @@ -607,25 +613,25 @@ def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None): Args: fn: A callable that will take as many arguments as there are passed iterables. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. - chunksize: The size of the chunks the iterable will be broken into - before being passed to a child process. This argument is only - used by ProcessPoolExecutor; it is ignored by + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. + chunksize: The size of the chunks the iterable will be broken + into before being passed to a child process. This argument + is only used by ProcessPoolExecutor; it is ignored by ThreadPoolExecutor. buffersize: The number of submitted tasks whose results have not - yet been yielded. If the buffer is full, iteration over the + yet been yielded. If the buffer is full, iteration over the iterables pauses until a result is yielded from the buffer. - If None, all input elements are eagerly collected, and a task is - submitted for each. + If None, all input elements are eagerly collected, and + a task is submitted for each. Returns: - An iterator equivalent to: map(func, *iterables) but the calls may - be evaluated out-of-order. + An iterator equivalent to: map(func, *iterables) but the calls + may be evaluated out-of-order. Raises: - TimeoutError: If the entire result iterator could not be generated - before the given timeout. + TimeoutError: If the entire result iterator could not be + generated before the given timeout. Exception: If fn(*args) raises for any values. """ if buffersize is not None and not isinstance(buffersize, int): @@ -679,8 +685,8 @@ def shutdown(self, wait=True, *, cancel_futures=False): Args: wait: If True then shutdown will not return until all running - futures have finished executing and the resources used by the - executor have been reclaimed. + futures have finished executing and the resources used by + the executor have been reclaimed. cancel_futures: If True then shutdown will cancel all pending futures. Futures that are completed or running will not be cancelled. diff --git a/Lib/concurrent/futures/interpreter.py b/Lib/concurrent/futures/interpreter.py index 53c6e757ded2e33..fd3d45144b49a71 100644 --- a/Lib/concurrent/futures/interpreter.py +++ b/Lib/concurrent/futures/interpreter.py @@ -2,7 +2,6 @@ from concurrent import interpreters import sys -import textwrap from . import thread as _thread import traceback @@ -111,8 +110,8 @@ def __init__(self, max_workers=None, thread_name_prefix='', """Initializes a new InterpreterPoolExecutor instance. Args: - max_workers: The maximum number of interpreters that can be used to - execute the given calls. + max_workers: The maximum number of interpreters that can be used + to execute the given calls. thread_name_prefix: An optional name prefix to give our threads. initializer: A callable or script used to initialize each worker interpreter. diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index a14650bf5fa47cd..10d4ac89d725713 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -469,17 +469,33 @@ def _terminate_broken(self, cause): executor._shutdown_thread = True executor = None - # All pending tasks are to be marked failed with the following - # BrokenProcessPool error - bpe = BrokenProcessPool("A process in the process pool was " - "terminated abruptly while the future was " - "running or pending.") + # All pending tasks are to be marked failed with a + # BrokenProcessPool error, as separate instances to avoid sharing + # a traceback (gh-101267). + cause_str = None if cause is not None: - bpe.__cause__ = _RemoteTraceback( - f"\n'''\n{''.join(cause)}'''") + cause_str = ''.join(cause) + else: + # No cause known, so report any processes that have + # terminated with nonzero exit codes, e.g. from a + # segfault. Multiple may terminate simultaneously, + # so include all of them in the traceback. + errors = [] + for p in self.processes.values(): + if p.exitcode is not None and p.exitcode != 0: + errors.append(f"Process {p.pid} terminated abruptly " + f"with exit code {p.exitcode}") + if errors: + cause_str = "\n".join(errors) + cause_tb = f"\n'''\n{cause_str}'''" if cause_str else None # Mark pending tasks as failed. for work_id, work_item in self.pending_work_items.items(): + bpe = BrokenProcessPool("A process in the process pool was " + "terminated abruptly while the future was " + "running or pending.") + if cause_tb is not None: + bpe.__cause__ = _RemoteTraceback(cause_tb) try: work_item.future.set_exception(bpe) except _base.InvalidStateError: @@ -642,19 +658,21 @@ def __init__(self, max_workers=None, mp_context=None, Args: max_workers: The maximum number of processes that can be used to - execute the given calls. If None or not given then as many - worker processes will be created as the machine has processors. - mp_context: A multiprocessing context to launch the workers created - using the multiprocessing.get_context('start method') API. This - object should provide SimpleQueue, Queue and Process. + execute the given calls. If None or not given then as many + worker processes will be created as the machine has + processors. + mp_context: A multiprocessing context to launch the workers + created using the multiprocessing.get_context('start method') + API. This object should provide SimpleQueue, Queue and + Process. initializer: A callable used to initialize worker processes. initargs: A tuple of arguments to pass to the initializer. - max_tasks_per_child: The maximum number of tasks a worker process - can complete before it will exit and be replaced with a fresh - worker process. The default of None means worker process will - live as long as the executor. Requires a non-'fork' mp_context - start method. When given, we default to using 'spawn' if no - mp_context is supplied. + max_tasks_per_child: The maximum number of tasks a worker + process can complete before it will exit and be replaced + with a fresh worker process. The default of None means + worker process will live as long as the executor. Requires + a non-'fork' mp_context start method. When given, we + default to using 'spawn' if no mp_context is supplied. """ _check_system_limits() @@ -824,24 +842,25 @@ def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None): Args: fn: A callable that will take as many arguments as there are passed iterables. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. - chunksize: If greater than one, the iterables will be chopped into - chunks of size chunksize and submitted to the process pool. - If set to one, the items in the list will be sent one at a time. + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. + chunksize: If greater than one, the iterables will be chopped + into chunks of size chunksize and submitted to the process + pool. If set to one, the items in the list will be sent + one at a time. buffersize: The number of submitted tasks whose results have not - yet been yielded. If the buffer is full, iteration over the + yet been yielded. If the buffer is full, iteration over the iterables pauses until a result is yielded from the buffer. - If None, all input elements are eagerly collected, and a task is - submitted for each. + If None, all input elements are eagerly collected, and + a task is submitted for each. Returns: - An iterator equivalent to: map(func, *iterables) but the calls may - be evaluated out-of-order. + An iterator equivalent to: map(func, *iterables) but the calls + may be evaluated out-of-order. Raises: - TimeoutError: If the entire result iterator could not be generated - before the given timeout. + TimeoutError: If the entire result iterator could not be + generated before the given timeout. Exception: If fn(*args) raises for any values. """ if chunksize < 1: diff --git a/Lib/concurrent/interpreters/__init__.py b/Lib/concurrent/interpreters/__init__.py index aa46a2b37a48d51..ea4147ee9a25da5 100644 --- a/Lib/concurrent/interpreters/__init__.py +++ b/Lib/concurrent/interpreters/__init__.py @@ -149,12 +149,17 @@ def __del__(self): def __reduce__(self): return (type(self), (self._id,)) - def _decref(self): + # gh-135729: Globals might be destroyed by the time this is called, so we + # need to keep references ourself + def _decref(self, *, + InterpreterNotFoundError=InterpreterNotFoundError, + _interp_decref=_interpreters.decref, + ): if not self._ownsref: return self._ownsref = False try: - _interpreters.decref(self._id) + _interp_decref(self._id) except InterpreterNotFoundError: pass diff --git a/Lib/concurrent/interpreters/_queues.py b/Lib/concurrent/interpreters/_queues.py index 9c12b2c8c246642..5f3ee0934de59d6 100644 --- a/Lib/concurrent/interpreters/_queues.py +++ b/Lib/concurrent/interpreters/_queues.py @@ -170,13 +170,13 @@ def full(self): def qsize(self): return _queues.get_count(self._id) - def put(self, obj, timeout=None, *, + def put(self, obj, block=True, timeout=None, *, unbounditems=None, _delay=10 / 1000, # 10 milliseconds ): """Add the object to the queue. - This blocks while the queue is full. + If "block" is true, this blocks while the queue is full. For most objects, the object received through Queue.get() will be a new one, equivalent to the original and not sharing any @@ -185,7 +185,8 @@ def put(self, obj, timeout=None, *, underlying data is actually shared. Furthermore, some types can be sent through a queue more efficiently than others. This group includes various immutable types like int, str, bytes, and - tuple (if the items are likewise efficiently shareable). See interpreters.is_shareable(). + tuple (if the items are likewise efficiently shareable). + See interpreters.is_shareable(). "unbounditems" controls the behavior of Queue.get() for the given object if the current interpreter (calling put()) is later @@ -209,6 +210,8 @@ def put(self, obj, timeout=None, *, If "unbounditems" is UNBOUND then it is returned by get() in place of the unbound item. """ + if not block: + return self.put_nowait(obj, unbounditems=unbounditems) if unbounditems is None: unboundop = -1 else: @@ -221,7 +224,7 @@ def put(self, obj, timeout=None, *, while True: try: _queues.put(self._id, obj, unboundop) - except QueueFull as exc: + except QueueFull: if timeout is not None and time.time() >= end: raise # re-raise time.sleep(_delay) @@ -235,17 +238,19 @@ def put_nowait(self, obj, *, unbounditems=None): unboundop, = _serialize_unbound(unbounditems) _queues.put(self._id, obj, unboundop) - def get(self, timeout=None, *, + def get(self, block=True, timeout=None, *, _delay=10 / 1000, # 10 milliseconds ): """Return the next object from the queue. - This blocks while the queue is empty. + If "block" is true, this blocks while the queue is empty. If the next item's original interpreter has been destroyed then the "next object" is determined by the value of the "unbounditems" argument to put(). """ + if not block: + return self.get_nowait() if timeout is not None: timeout = int(timeout) if timeout < 0: @@ -254,7 +259,7 @@ def get(self, timeout=None, *, while True: try: obj, unboundop = _queues.get(self._id) - except QueueEmpty as exc: + except QueueEmpty: if timeout is not None and time.time() >= end: raise # re-raise time.sleep(_delay) @@ -273,7 +278,7 @@ def get_nowait(self): """ try: obj, unboundop = _queues.get(self._id) - except QueueEmpty as exc: + except QueueEmpty: raise # re-raise if unboundop is not None: assert obj is None, repr(obj) diff --git a/Lib/configparser.py b/Lib/configparser.py index 18af1eadaad1110..a53ac87276445ad 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -315,12 +315,15 @@ def __init__(self, source, *args): def append(self, lineno, line): self.errors.append((lineno, line)) - self.message += '\n\t[line %2d]: %s' % (lineno, repr(line)) + self.message += f'\n\t[line {lineno:2d}]: {line!r}' def combine(self, others): + messages = [self.message] for other in others: - for error in other.errors: - self.append(*error) + for lineno, line in other.errors: + self.errors.append((lineno, line)) + messages.append(f'\n\t[line {lineno:2d}]: {line!r}') + self.message = "".join(messages) return self @staticmethod @@ -613,7 +616,9 @@ class RawConfigParser(MutableMapping): \] # ] """ _OPT_TMPL = r""" - (?P