A Few of My Favorite (Python) Things

A Few of My Favorite Things

Mike Pirnat • AG Interactive • CodeMash 2012

_ ( =
255,
lambda
V ,B,c
:c and Y(V*V+B,B, c
-1)if(abs(V)<6)else
( 2+c-4*abs(V)**-0.4)/i
) ;v, x=1500,1000;C=range(v*x
);import struct;P=struct.pack;M,
j ='<QIIHHHH',open('M.bmp','wb').write
for X in j('BM'+P(M,v*x*3+26,26,12,v,x,1,24))or C:
i ,Y=_;j(P('BBB',*(lambda T:(T*80+T**9
*i-950*T **99,T*70-880*T**18+701*
T **9 ,T*i**(1-T**45*2)))(sum(
[ Y(0,(A%3/3.+X%v+(X/v+
A/3/3.-x/2)/1j)*2.5
/x -2.7,i)**2 for
A in C
[:9]])
/9)
) )

http://preshing.com/20110926/high-resolution-mandelbrot-in-obfuscated-python

http://preshing.com/20110926/high-resolution-mandelbrot-in-obfuscated-python

The Interactive Shell

$ python
Python 2.7.1 (r271:86832, Jun 16 2011, 16:59:05)
[GCC 4.2.1 (Based on Apple Inc. build 5658) (LLVM build
2335.15.00)] on darwin
Type "help", "copyright", "credits" or "license" for
more information.
>>>

Docstrings...
class Frobulator(object):
"""It frobulates things."""

def __init__(self, doohickey):
"""A Frobulator needs a doohickey."""
self.doohickey = doohickey

def frobulate(self):
"""Frobulate ALL the things!"""
print "Frobulating..."
doodad = self.doohickey()
return FrobulatedThing(doodad)

class FrobulatedThing(object):
"""A thing which has been frobulated."""

def __init__(self, thing):
"""Make a thing into a frobulated thing."""
self.thing = thing

Comprehensions

• List comprehensions
• Set comprehensions
• Dictionary comprehensions
• Generator expressions

List Comprehensions

x = [item for item in series]

x = [do_something(item) for item in series if expression]

things = [Thingy.from_data(x) for x in database_results]

partiers = [x for x in codemashers if x.slides_done()]

List Comprehensions
booze = ['beer', 'wine', 'scotch', 'gin']
soft_drinks = ['water', 'soda', 'juice']

a = [(x, y) for x in booze for y in soft_drinks]

[('beer', 'water'), ('beer', 'soda'), ('beer', 'juice'),
('wine', 'water'), ('wine', 'soda'), ('wine', 'juice'),
('scotch', 'water'), ('scotch', 'soda'), ('scotch',
'juice'), ('gin', 'water'), ('gin', 'soda'), ('gin',
'juice')]

List Comprehensions

b = [x for x in zip(booze, soft_drinks)]

[('beer', 'water'), ('wine', 'soda'), ('scotch', 'juice')]

Set Comprehensions

s = {v for v in 'CODEMASH ROCKS' if v not in 'ABCD'}

set([' ', 'E', 'H', 'K', 'M', 'O', 'S', 'R'])

Dictionary Comprehensions
d = {key: value for key, value in list_of_tuples}

d = {
'Mike': 'Python',
'Jim': 'Ruby',
'Brad': 'UX',
...
}

d1 = {val: key for key, val in d.items()}

{'Python': 'Mike', 'Ruby': 'Jim', 'UX': 'Brad', ...}

Generators
def f(how_many):
for number in range(how_many):
if number**2 > 3:
yield number * 2

for number in f(5):
print number

4
6
8

Generator Expressions

gen = (2*x for x in range(5) if x**2 > 3)

for number in gen:
print number

4
6
8

Properties

class Foo(object):

def __init__(self, bar=42):
self.set_bar(bar)

def get_bar(self):
return self.bar

def set_bar(self, bar):
self.bar = int(bar)

Properties
class Foo(object):

def __init__(self, bar=42):
self.bar = bar

@property
def bar(self):
return self._bar

@bar.setter
def bar(self, x):
self._bar = int(x)

Properties

>>> foo = Foo()
>>> foo.bar
42
>>> foo.bar = 'abc'
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<stdin>", line 9, in bar
ValueError: invalid literal for int() with base 10:
'abc'

Decorators

• That funny-looking @ thing
• Similar to Java’s annotations
• Replaces a function with a wrapper
• Augment functionality in a reusable way

Decorators
def be_excellent(wrapped):
def wrapper(*args, **kwargs):
print "Be excellent to each other..."
return wrapped(*args, **kwargs)
return wrapper

@be_excellent
def party_on(who):
print "...and party on, {0}!".format(who)

>>> party_on('dudes')
Be excellent to each other...
...and party on, dudes!

Context Managers
• That funny “with” thing
• Simplify calling code that needs setup and
teardown
• Alternative to try/ﬁnally structures
• Acquire locks, open and close ﬁles, do
database transactions, etc.

Before Context Managers

frobulator = Frobulator(...)

try:
frobulator.frobulate()
finally:
frobulator.cleanup()

After Context Managers

with Frobulator(...) as frobulator:


def __init__(self, ...):
...

def __enter__(self):
print "Preparing to frobulate..."
return self

def __exit__(self, exc_type, exc_value, exc_tb):
print "Frobulation complete!"

print "Frobulating!"
...

with Frobulator() as frobulator:

Preparing to frobulate...
Frobulating!
Frobulation complete!

import contextlib

@contextlib.contextmanager
def make_frobulator():
print "Preparing to frobulate..."
yield Frobulator()
print "Frobulation complete!"

with make_frobulator() as frobulator:

Preparing to frobulate...
Frobulating!
Frobulation complete!

Itertools

• Tools for iterating on sequences
• Inspired by functional languages
• Fast and memory efﬁcient
• Combine to express more complicated
algorithms

Itertools: chain

from itertools import chain

for x in chain([1, 2, 3], [42, 1138, 2112]):
print x,

1 2 3 42 1138 2112

Itertools: izip

from itertools import izip

for x in izip([1, 2, 3], [42, 1138, 2112]):
print x,

(1, 42) (2, 1138) (3, 2112)

Itertools: islice
from itertools import islice, count

for x in islice(count(), 5):
print x,

for x in islice(count(), 5, 10):
print x,

for x in islice(count(), 0, 100, 10):
print x,

0 1 2 3 4
5 6 7 8 9
0 10 20 30 40 50 60 70 80 90

Itertools: imap

from itertools import imap

for thingy in imap(Thingy.from_data, database_results):
...

Itertools: cycle
from itertools import cycle

for x in cycle(['wake up', 'meet Ned', 'romance Rita']):
print x

wake up
meet Ned
romance Rita
wake up
meet Ned
romance Rita
wake up
meet Ned
...

Itertools: repeat

from itertools import repeat

for x in repeat("stop hitting yourself", 5):
print x

stop hitting yourself

Functools

• Tools for manipulating functions
• Partial
• Wraps a callable with default arguments
• Alternative to lambdas and closures

Functools: Partial
from functools import partial

def f(a, b=2):
print a, b

f1 = partial(f, 'fixed_a')
f2 = partial(f, b='fixed_b')

>>> f1(b=1138)
fixed_a 1138

>>> f2(1138)
1138 fixed_b

Functools: Partial
def category_is(category, item):
categories = [x.lower() for x in item.categories]
if category.lower() in categories:
return True
return False

is_python = partial(category_is, 'python')
is_cat_pictures = partial(category_is, 'catpix')

...

python_posts = [item for item in blog_posts if is_python(item)]

cat_posts = [item for item in blog_posts if is_cat_pictures(item)]

Collections
• Beyond the basic list, dict, tuple, and set...
• Counter
• Defaultdict
• OrderedDict
• Namedtuple
• ...and more

Counter
counter = {}
for char in "Hello there, CodeMash!":
if char not in counter:
counter[char] = 1
else:
counter[char] += 1

print counter

{'a': 1, ' ': 2, 'C': 1, 'e': 4, 'd': 1, 'H': 1, 'M': 1,
'l': 2, 'o': 2, ',': 1, 's': 1, 'r': 1, '!': 1, 't': 1,
'h': 2}

Counter

from collections import Counter
counter = Counter("Hello there, CodeMash!")
print counter

Counter({'e': 4, ' ': 2, 'l': 2, 'o': 2, 'h': 2, 'a': 1,
'C': 1, 'd': 1, 'H': 1, 'M': 1, ',': 1, 's': 1, 'r': 1,
'!': 1, 't': 1})

Namedtuple
import math

def distance(a, b):
return math.sqrt(
(a[0] - b[0])**2 +
(a[1] - b[1])**2 +
(a[2] - b[2])**2
)

a = (1, 2, 3)
b = (-1, -2, 42)
print distance(a, b)

39.25557285278104

Namedtuple
from collections import namedtuple
Point = namedtuple('Point', 'x y z')

def distance(a, b):
return math.sqrt(
(a.x - b.x)**2 +
(a.y - b.y)**2 +
(a.z - b.z)**2
)

a = Point(x=1, y=2, z=3)
b = Point(-1, -2, 42)
print distance(a, b)

39.25557285278104

Difﬂib
s1 = """
Lorem ipsum dolor sit amet, consectetur adipiscing elit.
Phasellus dui nunc, faucibus id ullamcorper eget, tempus
vitae nisl. Donec quis semper risus. Curabitur sit amet
tellus eget metus accumsan porta nec nec lorem. Ut vitae
sem nisl. Praesent pulvinar feugiat nibh fringilla
semper. Nullam cursus tempor lorem ut egestas. Nullam
suscipit gravida turpis ac porttitor. Curabitur eleifend
augue at risus commodo pretium. Aliquam eget magna
risus, ut lobortis metus. Cum sociis natoque penatibus
et magnis dis parturient montes, nascetur ridiculus mus.
Etiam non magna sit amet nulla porttitor molestie sit
amet vel sem. Vestibulum sit amet nisl a velit
adipiscing porta id non urna. Duis ullamcorper dictum
ipsum sit amet congue.
"""

Difﬂib
s2 = """
risus, ut lobortis montes. Cum sociis natoque penatibus
et magnis dis parturient metus, nascetur ridiculus mus.
"""

Difﬂib

import difflib

differ = difflib.Differ()
diff = differ.compare(s1.splitlines(), s2.splitlines())
print 'n'.join(diff)

Difﬂib
- risus, ut lobortis metus. Cum sociis natoque penatibus
? --

+ risus, ut lobortis montes. Cum sociis natoque penatibus
? +++

- et magnis dis parturient montes, nascetur ridiculus mus.
? ^^ ^

+ et magnis dis parturient metus, nascetur ridiculus mus.
? ^ ^


Difﬂib

diff = difflib.unified_diff(
s1.splitlines(), s2.splitlines(), lineterm='')
print 'n'.join(diff)

Difﬂib

---
+++
@@ -7,8 +7,8 @@
-risus, ut lobortis metus. Cum sociis natoque penatibus
-et magnis dis parturient montes, nascetur ridiculus mus.
+risus, ut lobortis montes. Cum sociis natoque penatibus
+et magnis dis parturient metus, nascetur ridiculus mus.

Ast

• Use Python to process abstract syntax trees
of Python grammar
• Helps introspect about what the current
grammar looks like
• Helps write secure code

Ast

# Danger:
foo = eval(bar)

# Safe:
foo = ast.literal_eval(bar)

Multiprocessing
• Like threading, but with subprocesses
• Local and remote concurrency
• Spawn processes or entire pools
• Communicate via queues or pipes
• Shared state via shared memory or
manager/server process

import os
from multiprocessing import Process

def info(title):
print title
print 'parent process:', os.getppid()
print 'process id:', os.getpid()

def f(name):
info('function f')
print 'hello', name

if __name__ == '__main__':
info('main')
p = Process(target=f, args=('world',))
p.start()
p.join()

$ python process.py
main
parent process: 18647
process id: 31317
----------
function f
process id: 31318
----------
hello world

import os
from multiprocessing import Pool

def g(x):
info('function g('+str(x)+')')
return x * x

if __name__ == '__main__':
info('main')
pool = Pool(2)
print pool.map(g, range(100))

$ python process.py parent process: 31369
main parent process: 31369
module name: __main__ process id: 31370
parent process: 18647 process id: 31371
process id: 31369 ----------
---------- ----------
function g(0) function g(4)
module name: __main__ module name: __main__
parent process: 31369 parent process: 31369
process id: 31370 function g(14)
---------- process id: 31370
function g(1) module name: __main__
module name: __main__ ----------
parent process: 31369 parent process: 31369
process id: 31370 function g(5)
---------- process id: 31371
function g(13) module name: __main__
function g(2)
module name: __main__ ...
process id: 31370 [0, 1, 4, 9, 16, 25, ..., 8836,
---------- 9025, 9216, 9409, 9604, 9801]
module name: __main__
function g(3)
module name: __main__

Third Party Packages

• Most available on PyPI--http://pypi.python.org
• pip install packagename
• easy_install packagename

Virtualenv

• Makes an isolated Python environment
• Don’t pollute your global site-packages
• Insulates Python projects from one another
• Don't have to be root

Virtualenv
$ virtualenv directory

$ virtualenv --python=/path/to/specific/python directory

$ cd directory
$ . bin/activate

$ easy_install whatever
$ pip install whatever

...do stuff...

$ deactivate

Datetime and Dateutil
• Python’s datetime provides date, time,
datetime, and timedelta objects
• Dateutil provides powerful extensions
• Parsing
• Olson-driven timezones
• Recurrence

Parsing

>>> from dateutil.parser import parse

>>> parse('01/01/2012')
datetime.datetime(2012, 1, 1, 0, 0)

>>> parse('2012-01-01')
datetime.datetime(2012, 1, 1, 0, 0)

Timezones
>>> from dateutil import zoneinfo

>>> zone = zoneinfo.gettz('US/Eastern')
>>> zone
tzfile('America/New_York')
>>> zone2 = zoneinfo.gettz('US/Hawaii')

>>> dt = datetime.now(zone)
>>> dt
datetime.datetime(2012, 1, 13, 13, 46, 54, 997825,
tzinfo=tzfile('America/New_York'))

>>> dt.astimezone(zone2)
datetime.datetime(2012, 1, 13, 8, 46, 54, 997825,
tzinfo=tzfile('Pacific/Honolulu'))

Recurrence
>>> from dateutil.rrule import rrule, YEARLY, MONTHLY,
WEEKLY, DAILY, HOURLY, MINUTELY, SECONDLY, BY_EASTER

>>> rr = rrule(YEARLY, dtstart=datetime(1948, 2, 24))

>>> rr.after(datetime(2012, 1, 1))
datetime(2012, 2, 24, 0, 0)

>>> rr.between(datetime(2010, 1, 1),
datetime(2012, 1, 1))
[datetime(2010, 2, 24, 0, 0),
datetime(2011, 2, 24, 0, 0)]

howoldismykid.com
def periods_between(freq, start_date, end_date):

    # dateutil.rrule falls down for monthly recurrences where the start
# date's day is greater than the number of days in a subsequent month
# in the range; ie, when start date is 10/31/2010, and end date is
# 3/13/2011, rrule's between will only produce 2 instances, 12/31 and
# 1/31, rather than 4.
    if freq == MONTHLY and start_date.day > 28:
        start_date = datetime.datetime(start_date.year, start_date.month,
                28, start_date.hour, start_date.minute, start_date.second)

    # Same problem but for "Pirates of Penzance" leap day birthdays...
    elif freq == YEARLY and is_leap_day(start_date):
        start_date = datetime.datetime(start_date.year, start_date.month,
                28, start_date.hour, start_date.minute, start_date.second)

    rr = rrule(freq, dtstart=start_date)
    periods = len(rr.between(start_date, end_date))
    return periods

howoldismykid.com
>>> start_date = datetime(2007, 9, 10)
>>> end_date = datetime(2012, 1, 13)

>>> periods_between(YEARLY, start_date, end_date)
4

>>> periods_between(MONTHLY, start_date, end_date)
52

>>> periods_between(WEEKLY, start_date, end_date)
226

>>> periods_between(DAILY, start_date, end_date)
1585

Nose

• Find, run, and report results of test suites
• Low-friction setup
• Extendable with plugins (more later)
• http://readthedocs.org/docs/nose/en/latest/

Nose

# test_foo.py

def test_a_thing():
assert False

def test_another_thing():
assert True

Nose
$ nosetests tests/test_foo.py
F.
======================================================================
FAIL: test_foo.test_a_thing
----------------------------------------------------------------------
Traceback (most recent call last):
File "/Users/mpirnat/Code/frobulator/lib/python2.7/site-packages/
nose-1.1.2-py2.7.egg/nose/case.py", line 197, in runTest
self.test(*self.arg)
File "/Users/mpirnat/Code/frobulator/src/frobulator/tests/test_foo.py",
line 2, in test_a_thing
assert False
AssertionError

----------------------------------------------------------------------
Ran 2 tests in 0.001s

FAILED (failures=1)

Nose
class TestWhenFrobulating(object):

def setup(self):
self.frobulator = Frobulator()
self.frobulator.frobulate()

def test_that_it_frobulated(self):
assert self.frobulator.has_frobulated

def teardown(self):
self.frobulator.cleanup()

Nose

$ nosetests
.
--------------------------------------------------------
Ran 1 test in 0.005s

OK

Nose: Useful Flags

• --processes=number
• -x
• --pdb
• --pdb-failures
• --failed

Nose Plugins
• Add custom behavior to your test run
• Get coverage statistics (coverage)
• Measure cleanliness with PEP-8 (tissue)
• Emit human-readable spec-style results
(pinocchio)
• Or write your own...

class TestGeneratesLotsOfFailures(object):

def test_generates_failures(self):

def _make_a_test(i):
# start with some wins
if i < 7:
assert True

# but then it hits the fan...
elif i < 30:
assert False

# then be a little random
elif i % 3 == 0:
assert False
else:
assert True

for i in range(50):
yield _make_a_test, i

$ nosetests test_epic_fail.py

.......FFFFFFFFFFFFFFFFFFFFFFFF..F..F..F..F..F..F.
[lots of test failure output; use your imagination...]
--------------------------------------------------------

FAILED (errors=30)

import os
from nose.plugins import Plugin

class F7U12(Plugin):

name = 'f7u12'
enabled = True

def options(self, parser, env=os.environ):
super(F7U12, self).options(parser, env=env)

def configure(self, options, config):
super(F7U12, self).configure(options, config)
self.config = config
if not self.enabled:
return

def setOutputStream(self, stream):
self.stream = stream
return self.stream

def begin(self):
self.failure_count = 0

def handleFailure(self, test, err):
self.failure_count += 1

if self.failure_count < 8:
self.stream.write('F')
else:
self.stream.write('U')
return True

$ nosetests --with-f7u12 test_epic_fail.py

.......FFFFFFFFUUUUUUUUUUUUUUUU..U..U..U..U..U..U.
[lots of test failure output; use your imagination...]
--------------------------------------------------------

FAILED (errors=30)

Mock
• Mock object framework
• Avoid writing custom stubs
• Uses “Action/Assert” model (not “Record/
Replay”)
• Can also patch out module and class
attributes in the scope of a test
• http://www.voidspace.org.uk/python/mock/

Mock

from mock import Mock

foo = Mock()
foo.bar()
foo.bar.baz(42)
foo.bar.baz.return_value = "whatever"

assert foo.bar.called
foo.bar.baz.assert_called_with(42)

Mock


def setup(self):
self.doohickey = Mock()
self.frobulator = Frobulator(self.doohickey)

def test_it_uses_the_doohickey(self):
assert self.doohickey.called

Mock
from mock import patch


@patch('frobulator.DingleHopper')
def setup(self, dingle_hopper_class):
self.dingle_hopper = Mock()
dingle_hopper_class.return_value =
self.dingle_hopper
self.frobulator = Frobulator()

def test_it_uses_a_dingle_hopper(self):
assert self.dingle_hopper.called

Coverage

• Measure coverage of your codebase during
program execution
• Integrates with several test runners to
measure test coverage
• http://nedbatchelder.com/code/coverage/

#!/usr/bin/env python

"""It frobulates things."""

def __init__(self, doohickey):
"""A Frobulator needs a doohickey."""
self.doohickey = doohickey

"""Frobulate ALL the things!"""
print "Frobulating..."
doodad = self.doohickey()
return FrobulatedThing(doodad)

class FrobulatedThing(object):
"""A thing which has been frobulated."""

def __init__(self, thing):
"""Make a thing into a frobulated thing."""
self.thing = thing

if __name__ == '__main__':
x = FrobulatedThing(42)

$ coverage run frobulator.py

$ coverage report -m
Name Stmts Miss Cover Missing
------------------------------------------
frobulator 12 4 67% 8, 12-14

from mock import Mock, patch
from frobulator import Frobulator, FrobulatedThing


def setup(self):
self.doohickey = Mock()
self.doohickey.return_value = 42

self.frobulator = Frobulator(self.doohickey)
self.result = self.frobulator.frobulate()

def test_it_uses_a_doohickey(self):
assert self.doohickey.called

def test_it_returns_a_frobulated_thing(self):
assert isinstance(self.result, FrobulatedThing)

$ nosetests --with-coverage --cover-package frobulator
..
Name Stmts Miss Cover Missing
------------------------------------------
frobulator 12 1 92% 26
------------------------------------------

OK

Useful Flags

• --cover-erase
• --cover-inclusive
• --cover-tests

A Few of My Favorite (Python) Things

A Few of My Favorite (Python) Things

More Related Content

What's hot

Viewers also liked

Similar to A Few of My Favorite (Python) Things

Recently uploaded

A Few of My Favorite (Python) Things