Python Advanced Concepts
Decorators, metaclasses, generators, async, and internals
IT & Tech Reports | Class IM24A | 2026
1. Decorators — In Depth
A decorator is a callable that takes a function (or class) and returns a modified version. Decorators are
syntactic sugar for the pattern func = decorator(func). They are used for logging, caching,
authentication, retry logic, and much more.
import functools, time, logging
# Basic decorator
def timer(func):
@[Link](func) # preserves __name__, __doc__
def wrapper(*args, **kwargs):
start = time.perf_counter()
result = func(*args, **kwargs)
elapsed = time.perf_counter() - start
print(f"{func.__name__} took {elapsed:.4f}s")
return result
return wrapper
@timer
def slow_function(n):
return sum(range(n))
# Decorator with arguments
def retry(max_attempts=3, delay=1.0, exceptions=(Exception,)):
def decorator(func):
@[Link](func)
def wrapper(*args, **kwargs):
last_exc = None
for attempt in range(1, max_attempts + 1):
try:
return func(*args, **kwargs)
except exceptions as e:
last_exc = e
[Link](f"{func.__name__} attempt {attempt} failed: {e}")
if attempt < max_attempts:
[Link](delay * attempt) # exponential backoff
raise last_exc
return wrapper
return decorator
@retry(max_attempts=5, delay=0.5, exceptions=(ConnectionError, TimeoutError))
def fetch_data(url: str) -> dict:
...
# Class-based decorator (can hold state)
class cache:
def __init__(self, func):
[Link] = func
[Link]: dict = {}
functools.update_wrapper(self, func)
def __call__(self, *args):
if args not in [Link]:
[Link][args] = [Link](*args)
return [Link][args]
def clear(self): [Link]()
@cache
def fibonacci(n: int) -> int:
if n < 2: return n
return fibonacci(n-1) + fibonacci(n-2)
2. Generators and Iterators
# Generator function — yields values lazily
def read_large_file(path: str, chunk_size: int = 8192):
with open(path, "r") as f:
while chunk := [Link](chunk_size):
yield chunk
# Generator expression (like list comprehension but lazy)
squares = (x**2 for x in range(10_000_000)) # no memory allocated yet
total = sum(squares) # computed on demand
# Infinite generator
def integers_from(start: int = 0):
n = start
while True:
yield n
n += 1
from itertools import islice
first_10 = list(islice(integers_from(5), 10)) # [5,6,7,...,14]
# send() — two-way communication with generator
def accumulator():
total = 0
while True:
value = yield total # yield current total, receive next value
if value is None: break
total += value
acc = accumulator()
next(acc) # prime the generator
[Link](10) # total = 10
[Link](20) # total = 30
# yield from — delegate to sub-generator
def chain(*iterables):
for it in iterables:
yield from it
list(chain([1,2], [3,4], [5])) # [1,2,3,4,5]
3. Context Managers
from contextlib import contextmanager, asynccontextmanager
import contextlib
# Class-based context manager
class Timer:
def __enter__(self):
[Link] = time.perf_counter()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
[Link] = time.perf_counter() - [Link]
return False # don't suppress exceptions
with Timer() as t:
slow_operation()
print(f"Elapsed: {[Link]:.3f}s")
# Generator-based context manager
@contextmanager
def managed_db_connection(dsn: str):
conn = connect(dsn)
try:
yield conn
[Link]()
except Exception:
[Link]()
raise
finally:
[Link]()
with managed_db_connection("postgresql://...") as conn:
[Link]("INSERT INTO ...")
# Suppress specific exceptions
with [Link](FileNotFoundError):
[Link]("temp_file.txt") # no error if file doesn't exist
# Async context manager
@asynccontextmanager
async def lifespan(app):
# startup
await [Link]()
yield
# shutdown
await [Link]()
4. Metaclasses and Descriptors
# Descriptor protocol — powers @property, ORM fields, etc.
class Validated:
def __set_name__(self, owner, name):
[Link] = name
[Link] = f"_{name}"
def __get__(self, obj, objtype=None):
if obj is None: return self # access via class
return getattr(obj, [Link], None)
def __set__(self, obj, value):
value = [Link](value)
setattr(obj, [Link], value)
def validate(self, value):
raise NotImplementedError
class PositiveFloat(Validated):
def validate(self, value):
if not isinstance(value, (int, float)) or value <= 0:
raise ValueError(f"{[Link]} must be a positive number, got {value}")
return float(value)
class Score(Validated):
def validate(self, value):
if not 1.0 <= float(value) <= 6.0:
raise ValueError(f"{[Link]} must be between 1 and 6")
return float(value)
class Student:
score = Score()
gpa = PositiveFloat()
def __init__(self, name, score):
[Link] = name
[Link] = score # triggers Score.__set__
# Metaclass — controls class creation
class SingletonMeta(type):
_instances: dict = {}
def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
cls._instances[cls] = super().__call__(*args, **kwargs)
return cls._instances[cls]
class DatabasePool(metaclass=SingletonMeta):
def __init__(self):
[Link] = []
5. Python Data Model — Dunder Methods
class Vector:
def __init__(self, x: float, y: float):
self.x, self.y = x, y
def __repr__(self) -> str:
return f"Vector({self.x}, {self.y})"
def __str__(self) -> str:
return f"({self.x}, {self.y})"
def __add__(self, other: "Vector") -> "Vector":
return Vector(self.x + other.x, self.y + other.y)
def __sub__(self, other: "Vector") -> "Vector":
return Vector(self.x - other.x, self.y - other.y)
def __mul__(self, scalar: float) -> "Vector":
return Vector(self.x * scalar, self.y * scalar)
__rmul__ = __mul__ # scalar * vector
def __abs__(self) -> float:
return (self.x**2 + self.y**2) ** 0.5
def __bool__(self) -> bool:
return bool(self.x or self.y)
def __eq__(self, other) -> bool:
return isinstance(other, Vector) and self.x==other.x and self.y==other.y
def __hash__(self) -> int:
return hash((self.x, self.y)) # needed for use in sets/dicts
def __iter__(self):
yield self.x; yield self.y # enables: x, y = vector
def __len__(self) -> int:
return 2
v1, v2 = Vector(1, 2), Vector(3, 4)
print(v1 + v2) # Vector(4, 6)
print(abs(v1)) # 2.236...
x, y = v1 # unpacking via __iter__
6. Abstract Base Classes and Protocols
from abc import ABC, abstractmethod
from typing import Protocol, runtime_checkable
# ABC — enforced interface via inheritance
class Repository(ABC):
@abstractmethod
async def find_by_id(self, id: int): ...
@abstractmethod
async def save(self, entity) -> None: ...
@abstractmethod
async def delete(self, id: int) -> None: ...
class PostgresUserRepository(Repository):
async def find_by_id(self, id: int):
return await [Link]("SELECT * FROM users WHERE id=$1", id)
async def save(self, user): ...
async def delete(self, id): ...
# Protocol — structural subtyping (duck typing with type checking)
@runtime_checkable
class Drawable(Protocol):
def draw(self) -> None: ...
def resize(self, factor: float) -> None: ...
class Circle:
def draw(self): print("Drawing circle")
def resize(self, f): [Link] *= f
# Works without inheriting from Drawable
assert isinstance(Circle(), Drawable) # True at runtime
7. Performance Profiling
import cProfile, pstats, io, timeit, tracemalloc
# timeit — benchmark small snippets
t = [Link]("[x**2 for x in range(1000)]", number=10000)
print(f"List comprehension: {t:.3f}s")
# cProfile — profile full programs
pr = [Link]()
[Link]()
my_function()
[Link]()
stream = [Link]()
ps = [Link](pr, stream=stream).sort_stats("cumulative")
ps.print_stats(20) # top 20 functions by cumulative time
print([Link]())
# tracemalloc — track memory allocations
[Link]()
expensive_operation()
snapshot = tracemalloc.take_snapshot()
for stat in [Link]("lineno")[:10]:
print(stat)
# line_profiler (pip install line_profiler)
# @profile decorator + kernprof -l -v [Link]
# memory_profiler (pip install memory_profiler)
# @profile decorator + python -m memory_profiler [Link]
8. Type Hints — Advanced
from typing import TypeVar, Generic, overload, Literal, TypeAlias, Never
from typing import get_type_hints
from dataclasses import dataclass
T = TypeVar("T")
K = TypeVar("K")
V = TypeVar("V")
# Generic class
class Stack(Generic[T]):
def __init__(self) -> None:
self._items: list[T] = []
def push(self, item: T) -> None:
self._items.append(item)
def pop(self) -> T:
return self._items.pop()
stack: Stack[int] = Stack()
# TypeAlias
Matrix: TypeAlias = list[list[float]]
JSON: TypeAlias = dict[str, "JSON"] | list["JSON"] | str | int | float | bool | None
# Literal types
Direction = Literal["north", "south", "east", "west"]
def move(direction: Direction, steps: int) -> None: ...
# Overload — different return types based on argument type
@overload
def parse(data: str) -> dict: ...
@overload
def parse(data: bytes) -> dict: ...
def parse(data):
if isinstance(data, bytes): data = [Link]()
return [Link](data)
# dataclass with frozen and slots
@dataclass(frozen=True, slots=True) # immutable + fast attribute access
class Point:
x: float
y: float
def distance(self) -> float:
return (self.x**2 + self.y**2) ** 0.5
9. Packaging and Project Structure
# Modern Python project layout
myproject/
[Link] # project metadata + build config (PEP 517/518)
[Link]
src/
myproject/
__init__.py
[Link]
[Link]
tests/
[Link]
test_core.py
docs/
# [Link] (using uv or pip)
[project]
name = "myproject"
version = "1.0.0"
requires-python = ">=3.12"
dependencies = [
"fastapi>=0.111",
"sqlalchemy>=2.0",
"pydantic>=2.0",
]
[[Link]-dependencies]
dev = ["pytest", "pytest-cov", "ruff", "mypy"]
[[Link]]
select = ["E", "F", "I", "N", "UP"]
[[Link]]
strict = true
python_version = "3.12"
[[Link].ini_options]
testpaths = ["tests"]
addopts = "--cov=src --cov-report=term-missing"
# Virtual environment and dependency management
python -m venv .venv && source .venv/bin/activate
pip install -e ".[dev]" # editable install with dev extras
10. CPython Internals
Understanding how CPython (the reference implementation) works helps write faster Python and debug
subtle issues.
Concept Details
Bytecode compilation Source → AST → bytecode (.pyc in __pycache__); dis module inspects bytecode
Concept Details
Reference counting Objects have refcount; freed when refcount reaches 0; cyclic refs handled by GC
Garbage collector Generational GC collects cyclic references (3 generations, threshold-based)
Small integer cache Integers -5 to 256 are cached singletons; 'is' comparison can be misleading
String interning Short strings and identifiers are interned (shared); [Link]() for others
Frame objects Each function call creates a frame object holding locals, stack, code object
GIL Mutex released every 5ms (switchinterval) or during I/O to allow context switches
PyPy JIT-compiled alternative — 5-10x faster for CPU-bound; different GC (no refcounting)