DTrace SystemTap
CPython
3.11.0
Guido van Rossum and the Python development team
01, 2022
Python Software Foundation
Email: docs@[Link]
Contents
1 2
2 DTrace 3
3 SystemTap 4
4 5
5 SystemTap Tapsets 6
6 7
David Malcolm
Łukasz Langa
DTrace SystemTap
•
•
•
Python 3.6 CPython DTrace
SystemTap CPython
CPython DTrace CPython CPython
CPython DTrace
1
1
macOS DTrace Linux SystemTap CPython
SystemTap
Linux
$ yum install systemtap-sdt-devel
$ sudo apt-get install systemtap-sdt-dev
CPython --with-dtrace :
checking for --with-dtrace... yes
macOS Python DTrace Python
$ python3.6 -q &
$ sudo dtrace -l -P python$! # or: dtrace -l -m python3.6
ID PROVIDER MODULE FUNCTION NAME
29564 python18035 python3.6 _PyEval_EvalFrameDefault function-entry
29565 python18035 python3.6 dtrace_function_entry function-entry
29566 python18035 python3.6 _PyEval_EvalFrameDefault function-
,→return
29567 python18035 python3.6 dtrace_function_return function-
,→return
29568 python18035 python3.6 collect gc-done
29569 python18035 python3.6 collect gc-start
29570 python18035 python3.6 _PyEval_EvalFrameDefault line
29571 python18035 python3.6 maybe_dtrace_line line
Linux .[Link] SystemTap
$ readelf -S ./python | grep .[Link]
[30] .[Link] NOTE 0000000000000000 00308d78
Python --enable-shared
:
$ readelf -S [Link].1.0 | grep .[Link]
[29] .[Link] NOTE 0000000000000000 00365b68
readelf
$ readelf -n ./python
Displaying notes found at file offset 0x00000254 with length 0x00000020:
Owner Data size Description
GNU 0x00000010 NT_GNU_ABI_TAG (ABI version tag)
OS: Linux, ABI: 2.6.32
Displaying notes found at file offset 0x00000274 with length 0x00000024:
Owner Data size Description
GNU 0x00000014 NT_GNU_BUILD_ID (unique build ID␣
,→bitstring)
Build ID: df924a2b08a7e89f6e11251d4602022977af2670
( )
2
( )
Displaying notes found at file offset 0x002d6c30 with length 0x00000144:
Owner Data size Description
stapsdt 0x00000031 NT_STAPSDT (SystemTap probe␣
,→descriptors)
Provider: python
Name: gc__start
Location: 0x00000000004371c3, Base: 0x0000000000630ce2, Semaphore:␣
,→0x00000000008d6bf6
Arguments: -4@%ebx
stapsdt 0x00000030 NT_STAPSDT (SystemTap probe␣
,→descriptors)
Provider: python
Name: gc__done
Location: 0x00000000004374e1, Base: 0x0000000000630ce2, Semaphore:␣
,→0x00000000008d6bf8
Arguments: -8@%rax
stapsdt 0x00000045 NT_STAPSDT (SystemTap probe␣
,→descriptors)
Provider: python
Name: function__entry
Location: 0x000000000053db6c, Base: 0x0000000000630ce2, Semaphore:␣
,→0x00000000008d6be8
Arguments: 8@%rbp 8@%r12 -4@%eax
stapsdt 0x00000046 NT_STAPSDT (SystemTap probe␣
,→descriptors)
Provider: python
Name: function__return
Location: 0x000000000053dba8, Base: 0x0000000000630ce2, Semaphore:␣
,→0x00000000008d6bea
Arguments: 8@%rbp 8@%r12 -4@%eax
The above metadata contains information for SystemTap describing how it can patch strategically placed machine
code instructions to enable the tracing hooks used by a SystemTap script.
2 DTrace
DTrace Python / ”start”
self int indent;
python$target:::function-entry
/copyinstr(arg1) == "start"/
{
self->trace = 1;
}
python$target:::function-entry
/self->trace/
{
printf("%d\t%*s:", timestamp, 15, probename);
printf("%*s", self->indent, "");
printf("%s:%s:%d\n", basename(copyinstr(arg0)), copyinstr(arg1), arg2);
self->indent++;
}
python$target:::function-return
( )
3
( )
/self->trace/
{
self->indent--;
printf("%d\t%*s:", timestamp, 15, probename);
printf("%*s", self->indent, "");
printf("%s:%s:%d\n", basename(copyinstr(arg0)), copyinstr(arg1), arg2);
}
python$target:::function-return
/copyinstr(arg1) == "start"/
{
self->trace = 0;
}
:
$ sudo dtrace -q -s call_stack.d -c "python3.6 [Link]"
:
156641360502280 function-entry:call_stack.py:start:23
156641360518804 function-entry: call_stack.py:function_1:1
156641360532797 function-entry: call_stack.py:function_3:9
156641360546807 function-return: call_stack.py:function_3:10
156641360563367 function-return: call_stack.py:function_1:2
156641360578365 function-entry: call_stack.py:function_2:5
156641360591757 function-entry: call_stack.py:function_1:1
156641360605556 function-entry: call_stack.py:function_3:9
156641360617482 function-return: call_stack.py:function_3:10
156641360629814 function-return: call_stack.py:function_1:2
156641360642285 function-return: call_stack.py:function_2:6
156641360656770 function-entry: call_stack.py:function_3:9
156641360669707 function-return: call_stack.py:function_3:10
156641360687853 function-entry: call_stack.py:function_4:13
156641360700719 function-return: call_stack.py:function_4:14
156641360719640 function-entry: call_stack.py:function_5:18
156641360732567 function-return: call_stack.py:function_5:21
156641360747370 function-return:call_stack.py:start:28
3 SystemTap
SystemTap
SystemTap Python /
probe process("python").mark("function__entry") {
filename = user_string($arg1);
funcname = user_string($arg2);
lineno = $arg3;
printf("%s => %s in %s:%d\\n",
thread_indent(1), funcname, filename, lineno);
}
probe process("python").mark("function__return") {
filename = user_string($arg1);
funcname = user_string($arg2);
lineno = $arg3;
( )
4
( )
printf("%s <= %s in %s:%d\\n",
thread_indent(-1), funcname, filename, lineno);
}
$ stap \
[Link] \
-c "./python [Link]"
11408 python(8274): => __contains__ in Lib/_abcoll.py:362
11414 python(8274): => __getitem__ in Lib/[Link]
11418 python(8274): => encode in Lib/[Link]
11424 python(8274): <= encode in Lib/[Link]
11428 python(8274): <= __getitem__ in Lib/[Link]
11433 python(8274): <= __contains__ in Lib/_abcoll.py:366
•
•
• PID
/
CPython --enable-shared libpython probe
:
probe process("python").mark("function__entry") {
probe process("python").library("[Link].1.0").mark("function__entry") {
( CPython 3.6 )
function__entry(str filename, str funcname, int lineno)
Python Python
$arg1, $arg2, $arg3
• $arg1 : (const char *) user_string($arg1)
• $arg2 : (const char *) user_string($arg2)
• $arg3 : int
function__return(str filename, str funcname, int lineno)
function__entry() Python ( return
) Python ( )
function__entry()
5
line(str filename, str funcname, int lineno)
Python Python C
function__entry()
gc__start(int generation)
Python arg0 [Link]()
gc__done(long collected)
Python arg0
import__find__load__start(str modulename)
importlib arg0
3.7 .
import__find__load__done(str modulename, int found)
importlib find_and_load arg0 arg1
3.7 .
audit(str event, void *tuple)
[Link]() PySys_Audit() arg0 C arg1
PyObject
3.8 .
5 SystemTap Tapsets
SystemTap ”tapset” SystemTap
CPython tapset
/*
Provide a higher-level wrapping around the function__entry and
function__return markers:
\*/
probe [Link] = process("python").mark("function__entry")
{
filename = user_string($arg1);
funcname = user_string($arg2);
lineno = $arg3;
frameptr = $arg4
}
probe [Link] = process("python").mark("function__return")
{
filename = user_string($arg1);
funcname = user_string($arg2);
lineno = $arg3;
frameptr = $arg4
}
SystemTap tapset “/usr/share/systemtap/tapset“
[Link](str filename, str funcname, int lineno, frameptr)
Python Python
6
[Link](str filename, str funcname, int lineno, frameptr)
[Link] Python
return Python
SystemTap tapset Python
probe [Link]
{
printf("%s => %s in %s:%d\n",
thread_indent(1), funcname, filename, lineno);
}
probe [Link]
{
printf("%s <= %s in %s:%d\n",
thread_indent(-1), funcname, filename, lineno);
}
The following script uses the tapset above to provide a top-like view of all running CPython code, showing the top
20 most frequently entered bytecode frames, each second, across the whole system:
global fn_calls;
probe [Link]
{
fn_calls[pid(), filename, funcname, lineno] += 1;
}
probe [Link](1000) {
printf("\033[2J\033[1;1H") /* clear screen \*/
printf("%6s %80s %6s %30s %6s\n",
"PID", "FILENAME", "LINE", "FUNCTION", "CALLS")
foreach ([pid, filename, funcname, lineno] in fn_calls- limit 20) {
printf("%6d %80s %6d %30s %6d\n",
pid, filename, lineno, funcname,
fn_calls[pid, filename, funcname, lineno]);
}
delete fn_calls;
}