Коварный CodeType,

или от segfault'а

к работающему коду

Андрей Захаревич

Дмитрий Алимов

С чего все началось

# Plain functions if inspect.isroutine(obj): if inspect.ismethod(obj): has_self = True argspec = inspect.getargspec(obj) # Class constructors elif inspect.isclass(obj): try: argspec = inspect.getargspec(obj.__new__) except (AttributeError, TypeError): argspec = inspect.getargspec(obj.__init__) has_self = True # obj objects elif hasattr(obj, '__call__'): method = obj.__call__ argspec = inspect.getargspec(method) has_self = inspect.ismethod(method) return argspec.args[1:] if has_self else args

Проблема

Моя функция принимает переменное число аргументов (*args)

Число передаваемых аргументов я узнаю в рантайме

Объект функции в это время уже создан

Что же делать?

Метапрограммирование!

Ленивый вариант

compile('def func(a, b): f(a, b)', 'func.py', 'exec')

Не спортивно

Идем на stackoverflow

import types def create_function(name, args): def y(): pass y_code = types.CodeType(args, y.func_code.co_nlocals, y.func_code.co_stacksize, y.func_code.co_flags, y.func_code.co_code, y.func_code.co_consts, y.func_code.co_names, y.func_code.co_varnames, y.func_code.co_filename, name, y.func_code.co_firstlineno, y.func_code.co_lnotab) return types.FunctionType(y_code, y.func_globals, name) myfunc = create_function('myfunc', 3) myfunc(1, 2, 3, 4) # TypeError: myfunc() takes exactly 3 arguments (4 given)

Кажется должно работать

def create_function(name, nargs): def y(*args): print args y_code = types.CodeType(nargs, y.func_code.co_nlocals, # Параметры ) return types.FunctionType(y_code, y.func_globals, name) myfunc = create_function('myfunc', 3) myfunc(1, 2, 3)

Но нет

[1] 8514 segmentation fault python func.py

Еще попытка

class CustomCall(object):

__metaclass__ = Meta

def __call__(self, *args):

self.call(*args)

def ret_func(self):

print self.call.__name__

obj = CustomCall()

obj.call(1, 2, 3)

# Если закомментировать, то все

упадет

obj(1, 2, 3)

class Meta(type): def __new__(mcls, name, bases, attrs): nargs = 3 varnames = tuple(['self'] + [str(i) for i in range(nargs)]) ret_code = types.CodeType( nargs + 1, attrs['ret_func'].func_code.co_nlocals, attrs['ret_func'].func_code.co_stacksize, attrs['ret_func'].func_code.co_flags, attrs['ret_func'].func_code.co_code, attrs['ret_func'].func_code.co_consts, attrs['ret_func'].func_code.co_names, varnames, attrs['ret_func'].func_code.co_filename, 'call', attrs['ret_func'].func_code.co_firstlineno, attrs['ret_func'].func_code.co_lnotab) attrs['call'] = types.FunctionType( ret_code, attrs['ret_func'].func_globals, 'call') return super(Meta, mcls).__new__(mcls, name, bases, attrs)

class CustomCall(object): __metaclass__ = Meta def __call__(self, *args): self.call(*args) def ret_func(self): print self.call.__name__ obj = CustomCall()

obj.call(1, 2, 3) # Если закомментировать, то все упадет obj(1, 2, 3)

Еще попытка

class CustomCall(object):

__metaclass__ = Meta

def __call__(self, *args):

self.call(*args)

def ret_func(self):

print self.call.__name__

obj = CustomCall()

obj.call(1, 2, 3)

# Если закомментировать, то все

упадет

obj(1, 2, 3)

class Meta(type): def __new__(mcls, name, bases, attrs): nargs = 3 varnames = tuple(['self'] + [str(i) for i in range(nargs)]) ret_code = types.CodeType( nargs + 1, attrs['ret_func'].func_code.co_nlocals, attrs['ret_func'].func_code.co_stacksize, attrs['ret_func'].func_code.co_flags, attrs['ret_func'].func_code.co_code, attrs['ret_func'].func_code.co_consts, attrs['ret_func'].func_code.co_names, varnames, attrs['ret_func'].func_code.co_filename, 'call', attrs['ret_func'].func_code.co_firstlineno, attrs['ret_func'].func_code.co_lnotab) attrs['call'] = types.FunctionType( ret_code, attrs['ret_func'].func_globals, 'call') return super(Meta, mcls).__new__(mcls, name, bases, attrs)

class CustomCall(object): __metaclass__ = Meta def __call__(self, *args): self.call(*args) def ret_func(self): print self.call.__name__ obj = CustomCall()

obj.call(1, 2, 3) # Если закомментировать, то все упадет obj(1, 2, 3)

Заглянем в Disassembler

5D09E429 |. 8B7424 10 MOV ESI,DWORD PTR SS:[ESP+10] ; ESI = frame

5D09E42D |. 8B7D 14 MOV EDI,DWORD PTR SS:[EBP+14] ; EDI = *args

5D09E430 |. 8B5C24 18 MOV EBX,DWORD PTR SS:[ESP+18] ; EBX = argcount

5D09E434 |. 81C6 38010000 ADD ESI,0x138 ; ESI += 0x138 (f_localsplus)

5D09E43A |. 2BFE SUB EDI,ESI ; EDI -= ESI

5D09E43C |. 8D6424 00 LEA ESP,[ESP] ; NOP

5D09E440 |> 8B0437 /MOV EAX,DWORD PTR DS:[ESI+EDI] ; EAX = *(ESI + EDI)

5D09E443 |. FF00 |INC DWORD PTR DS:[EAX] ; Py_INCREF(*EAX)

5D09E445 |. 8B0E |MOV ECX,DWORD PTR DS:[ESI] ; ECX = *ESI

5D09E447 |. 8906 |MOV DWORD PTR DS:[ESI],EAX ; *ESI = EAX

5D09E449 |. 85C9 |TEST ECX,ECX ; test ECX:

5D09E44B |.- 74 11 |JZ SHORT 5D09E45E ; if 0: goto 5D09E45E

5D09E44D |. 8301 FF |ADD DWORD PTR DS:[ECX],-1 ; else: Py_XDECREF(*ECX)

5D09E450 |.- 75 0C |JNZ SHORT 5D09E45E ; if not 0: goto 5D09E45E

5D09E452 |. 8B41 04 |MOV EAX,DWORD PTR DS:[ECX+4] ; ECX->ob_type

5D09E455 |. 51 |PUSH ECX ; save fastlocals[i] to stack

5D09E456 |. 8B48 18 |MOV ECX,DWORD PTR DS:[EAX+18] ; ECX = ob_type->tp_dealloc

5D09E459 |. FFD1 |CALL ECX ; call ob_type->tp_dealloc()

5D09E45B |. 83C4 04 |ADD ESP,4 ; restore ESP

5D09E45E |> 83C6 04 |ADD ESI,4 ; ESI += 4

5D09E461 |. 83EB 01 |SUB EBX,1 ; EBX -= 1

5D09E464 |.- 75 DA \JNZ SHORT 5D09E440 ;

01eb3030 00000001 ob_refcnt

01eb3034 5886d830 python27!PyFrame_Type

01eb3038 00000002 ob_size

01eb303c 023b6b30 f_back

01eb3040 023a6b18 f_code

01eb3044 01e790c0 f_builtins

01eb3048 01e8aa50 f_globals

01eb304c 00000000 f_locals

01eb3050 01eb316c f_valuestack

01eb3054 01eb316c f_stacktop

01eb3058 00000000 f_trace

01eb305c 00000000 f_exc_type

01eb3060 00000000 f_exc_value

01eb3064 00000000 f_exc_traceback

01eb3068 01f72cf0 f_tstate

01eb306c ffffffff f_lasti

01eb3070 0000002f f_lineno

01eb3074 00000000 f_iblock

01eb3078 baadf00d f_blockstack[20] = {b_type,

01eb307c baadf00d b_handler

01eb3080 baadf00d b_level}

...

01eb3168 023bf550 f_localsplus // frame + 0x138

01eb316c 01f7d8a0

01eb3170 baadf00d

01eb3174 baadf00d

Что в памяти

PyObject *

PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals,

PyObject **args, int argcount, PyObject **kws, int kwcount,

PyObject **defs, int defcount, PyObject *closure) {

...

n = co->co_argcount;

for (i = 0; i < n; i++) {

x = args[i];

Py_INCREF(x);

SETLOCAL(i, x);

}

...

https://github.com/python/cpython/blob/2.7/Python/ceval.c

#define Py_INCREF(op) ( \

_Py_INC_REFTOTAL _Py_REF_DEBUG_COMMA \

((PyObject*)(op))->ob_refcnt++)

#define GETLOCAL(i) (fastlocals[i])

#define SETLOCAL(i, value) do { PyObject *tmp = GETLOCAL(i); \

GETLOCAL(i) = value; \

Py_XDECREF(tmp); } while (0)

#define Py_XDECREF(op) do { if ((op) == NULL); else Py_DECREF(op); } while (0)

#define Py_DECREF(op) \

do { \

if (_Py_DEC_REFTOTAL _Py_REF_DEBUG_COMMA \

--((PyObject*)(op))->ob_refcnt != 0) \

_Py_CHECK_REFCNT(op) \

else \

_Py_Dealloc((PyObject *)(op)); \

} while (0)

#define _Py_Dealloc(op) ( \

_Py_INC_TPFREES(op) _Py_COUNT_ALLOCS_COMMA \

(*Py_TYPE(op)->tp_dealloc)((PyObject *)(op)))

#endif /* !Py_TRACE_REFS */

https://github.com/python/cpython/blob/2.7/Include/object.h

typedef struct _frame {

PyObject_VAR_HEAD

struct _frame *f_back; /* previous frame, or NULL */

PyCodeObject *f_code; /* code segment */

PyObject *f_builtins; /* builtin symbol table (PyDictObject) */

PyObject *f_globals; /* global symbol table (PyDictObject) */

PyObject *f_locals; /* local symbol table (any mapping) */

PyObject **f_valuestack; /* points after the last local */

PyObject **f_stacktop;

PyObject *f_trace; /* Trace function */

PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;

PyThreadState *f_tstate;

int f_lasti; /* Last instruction if called */

int f_lineno; /* Current line number */

int f_iblock; /* index in f_blockstack */

PyTryBlock f_blockstack[CO_MAXBLOCKS]; /* for try and loop blocks */

PyObject *f_localsplus[1]; /* locals+stack, dynamically sized */

} PyFrameObject;

https://github.com/python/cpython/blob/2.7/Include/frameobject.h

ncells = PyTuple_GET_SIZE(code->co_cellvars);

nfrees = PyTuple_GET_SIZE(code->co_freevars);

extras = code->co_stacksize + code->co_nlocals + ncells + nfrees;

if (free_list == NULL) {

f = PyObject_GC_NewVar(PyFrameObject, &PyFrame_Type, extras);

...

https://github.com/python/cpython/blob/2.7/Objects/frameobject.c

co_cellvars – is a tuple with the names of local variables referenced by nested functions;

co_freevars – names of variables in the function, defined in an enclosing function scope;

Память для f_localsplus

Проблема найдена

ret_code = types.CodeType( nargs + 1, attrs['ret_func'].func_code.co_nlocals, attrs['ret_func'].func_code.co_stacksize, attrs['ret_func'].func_code.co_flags, attrs['ret_func'].func_code.co_code, attrs['ret_func'].func_code.co_consts, attrs['ret_func'].func_code.co_names, varnames, attrs['ret_func'].func_code.co_filename, 'call', attrs['ret_func'].func_code.co_firstlineno, attrs['ret_func'].func_code.co_lnotab, (), # freevars () # cellvars )

Попробуем применить

def create_func(fun, nargs): def wrapper(): args = map(itemgetter(1), sorted(((var_name, arg) for (var_name, arg) in locals().items() if var_name != 'fun'), key=itemgetter(0))) fun(*args) varnames = tuple(str(i) for i in range(0, nargs)) ret_code = types.CodeType(nargs, wrapper.func_code.co_nlocals + nargs, # Параметры ) return types.FunctionType(ret_code, wrapper.func_globals, wrapper.func_code.co_name) def test_func(*args): print args fun = create_func(test_func, 3) fun(1, 2, 3)

Как-то не очень

Traceback (most recent call last):

File "func.py", line 38, in <module>

fun(1,2,3)

File "func.py", line 16, in wrapper

fun(*args)

SystemError: Objects/cellobject.c:24: bad argument to internal function

Заработало!

def create_func(fun, nargs): def wrapper(fun): args = map(itemgetter(1), sorted(((var_name, arg) for (var_name, arg) in locals().items() if var_name != 'fun'), key=itemgetter(0))) fun(*args) varnames = tuple(['fun'] + [str(i) for i in range(0, nargs)]) ret_code = types.CodeType(nargs + 1, wrapper.func_code.co_nlocals + nargs, # Параметры ) return partial(types.FunctionType(ret_code, wrapper.func_globals, wrapper.func_code.co_name), fun) def test_func(*args): print args fun = create_func(test_func, 3) fun(1, 2, 3)

Что-то потеряли

>>> dir(wrapper.func_code)

['__class__', '__cmp__', '__delattr__', '__doc__', '__eq__', '__format__', '__ge__',

'__getattribute__', '__gt__', '__hash__', '__init__', '__le__', '__lt__', '__ne__', '__new__',

'__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__',

'__subclasshook__', 'co_argcount', 'co_cellvars', 'co_code', 'co_consts', 'co_filename',

'co_firstlineno', 'co_flags', 'co_freevars', 'co_lnotab', 'co_name', 'co_names', 'co_nlocals',

'co_stacksize', 'co_varnames']

Добавим параметр

ret_code = types.CodeType(nargs, wrapper.func_code.co_nlocals + nargs, # Параметры wrapper.func_code.co_freevars) return types.FunctionType(ret_code, wrapper.func_globals, wrapper.func_code.co_name) >>>Traceback (most recent call last): File "func.py", line 39, in <module> fun = create_func(test_func, 3) File "func.py", line 33, in create_func return types.FunctionType(ret_code, wrapper.func_globals, wrapper.func_code.co_name) TypeError: arg 5 (closure) must be tuple

def create_func(fun, nargs): def wrapper(): args = map(itemgetter(1), sorted(((var_name, arg) for (var_name, arg) in locals().items() if var_name != 'fun'), key=itemgetter(0))) fun(*args) varnames = tuple([str(i) for i in range(0, nargs)]) ret_code = types.CodeType(nargs, wrapper.func_code.co_nlocals + nargs, # Параметры wrapper.func_code.co_freevars) return types.FunctionType(ret_code, wrapper.func_globals, wrapper.func_code.co_name, None, wrapper.func_closure) def test_func(*args): print args fun = create_func(test_func, 3) fun(1, 2, 3)

И наконец, замыкания работают!

… почти

from operator import itemgetter def wrapper(): print locals() args = map(itemgetter(1), sorted(((var_name, arg) for (var_name, arg) in locals().items() if var_name != 'fun'), key=itemgetter(0))) print locals() fun(*args)

{'1': 2, '0': 1, '2': 3, 'fun': <function test_func at 0x10cd62398>}

{'1': 2, '0': [1, 2, 3], '2': 3, 'fun': <function test_func at 0x10cd62398>}

Спасибо за внимание!

Вопросы?

SPb Python Interest Group

https://telegram.me/spbpython