I would like to split a pyx file which contains the declaration and definition of a class hierarchy into a pxd and pyx file for further use in another cython module.
Here is my pxd file (instrument.pxd):
cdef struct s_detector:
int size_x
int size_y
int offset
int board_id_max
ctypedef s_detector Detector
cdef class Instrument:
cdef int nbdet
cdef Detector* detectors
cdef class D11Instrument(Instrument):
pass
and here is my pyx file (instrument.pyx):
from libc.stdlib cimport free, malloc
from instrument cimport Detector, DetectorDescr, PixelInfo
cdef class Instrument:
def __dealloc__(self):
free(self.detectors)
cdef class D11Instrument(Instrument):
def __cinit__(self):
self.nbdet = 3
self.detectors = <Detector*>malloc(self.nbdet*sizeof(Detector))
self.detectors[0] = [256,192,0,6]
self.detectors[1] = [32,256,49152,7]
self.detectors[2] = [32,256,57344,8]
When compiling those files, I get the following kind of errors:
Error compiling Cython file:
------------------------------------------------------------
...
cdef class Instrument:
def __dealloc__(self):
free(self.detectors)
^
------------------------------------------------------------
extensions/instrument.pyx:11:17: Cannot convert Python object to 'void *'
Error compiling Cython file:
------------------------------------------------------------
...
self.detectors = <Detector*>malloc(self.nbdet*sizeof(Detector))
^
------------------------------------------------------------
extensions/instrument.pyx:48:25: Cannot convert 'Detector *' to Python object
Would you know what I am doing wrong with my pxd/pyx files ?
Stuck on some basic Cython here - what's a canonical and efficient way to define an an array of strings in Cython? Specifically, I want to define a fixed-length constant array of char. (Please note that I would prefer not to bring in NumPy at this point.)
In C this would be:
/* cletters.c */
#include <stdio.h>
int main(void)
{
const char *headers[3] = {"to", "from", "sender"};
int i;
for (i = 0; i < 3; i++)
printf("%s\n", headers[i]);
}
Attempt in Cython:
# cython: language_level=3
# letters.pyx
cpdef main():
cdef const char *headers[3] = {"to", "from", "sender"}
print(headers)
However, this gives:
(cy) $ python3 ./setup.py build_ext --inplace --quiet
cpdef main():
cdef const char *headers[3] = {"to", "from", "sender"}
^
------------------------------------------------------------
letters.pyx:5:32: Syntax error in C variable declaration
You need two lines:
%%cython
cpdef main():
cdef const char *headers[3]
headers[:] = ['to','from','sender`]
print(headers)
Somewhat counterintuitive is than one assigns unicode-strings (Python3!) to char*. That is one of Cython's quirks. On the other hand, while initializing everything with only one value, bytes-object is needed:
%%cython
cpdef main():
cdef const char *headers[3]
headers[:] = b'init_value` ## unicode-string 'init_value' doesn't work.
print(headers)
Another alternative is the following oneliner:
%%cython
cpdef main():
cdef const char **headers=['to','from','sender`]
print(headers[0], headers[1], headers[2])
which is not exactly the same as above and leads to the following C-code:
char const **__pyx_v_headers;
...
char const *__pyx_t_1[3];
...
__pyx_t_1[0] = ((char const *)"to");
__pyx_t_1[1] = ((char const *)"from");
__pyx_t_1[2] = ((char const *)"sender");
__pyx_v_headers = __pyx_t_1;
__pyx_v_headers is of type char ** and downside is, that print(headers)no longer works out of the box.
For python3 Unicode strings, this is possible-
cdef Py_UNICODE* x[2]
x = ["hello", "worlᏪd"]
or
cdef Py_UNICODE** x
x = ["hello", "worlᏪd"]
I want to use a library that gives me a dynamic array. The dynamic array struct has a property void* _heap_ptr which gives the start of the array.
After having built the list, I want to access this pointer in cython (to make a copy of the array). But I cannot seem to get the pointer element from the struct.
Here is my pyx:
cimport src.clist as l
def main():
cdef l.ptr_list basic_list
cdef int i = 42
basic_list = l.create_list_size(sizeof(i), 100)
l.list_add_ptr(basic_list, &i)
cdef int* arr;
arr = basic_list._heap_ptr
for i in range(1):
print(arr[i])
This is the error message:
Error compiling Cython file:
------------------------------------------------------------
...
l.list_add_ptr(basic_list, &i)
cdef int* arr;
arr = basic_list._heap_ptr
^
------------------------------------------------------------
src/test.pyx:14:20: Cannot convert Python object to 'int *'
Error compiling Cython file:
------------------------------------------------------------
...
l.list_add_ptr(basic_list, &i)
cdef int* arr;
arr = basic_list._heap_ptr
^
------------------------------------------------------------
src/test.pyx:14:20: Storing unsafe C derivative of temporary Python reference
And my pxd:
cdef extern from "src/list.h":
ctypedef struct _list:
void* _heap_ptr
ctypedef struct ptr_list:
pass
ptr_list create_list_size(size_t size, int length)
list_destroy(ptr_list this_list)
void* list_at_ptr(ptr_list this_list, int index)
list_add_ptr(ptr_list this_list, void* value)
How can I fix my code? Why is this happening? From my investigations that error message pops up if you have forgotten to declare something as C (ie. use malloc not libc.stdlib.malloc, but I cannot see that anything similar is happening here.)
There are two issues in your code.
First: struct ptr_list has no members and thus no member _heap_ptr. It probably should have been
ctypedef struct ptr_list:
void* _heap_ptr
Cython's error message is not really helpful here, but as you said it pops up usually when a C-declaration is forgotten.
Second: you need to cast from void * to int * explicitly:
arr = <int*>basic_list._heap_ptr
I'm having trouble compiling a particular source file in C from an R package (fastclime) so that I can use it as a Python module. This is my code to extract the R package online:
from __future__ import division
import os
import sys
import glob
if not os.path.exists('fastlp.h'):
! wget http://cran.r-project.org/src/contrib/fastclime_1.2.4.tar.gz
! tar -xzvf fastclime_1.2.4.tar.gz
Within the R package (in the [src] folder) is the source file [parametric.c]. I'd like to compile this in Python as a module with numpy inputs. Since I can't immediately test the the functionality of this code, I first tried to compile the [fastlp.c] source file. With fastlp, I can create a toy example to test if I successfully compiled a version of fastlp in Python. Then I can probably extend the steps to compile [parametric.c]
Here is my attempt:
1) First I create the header file since it does not exist
%%file fastclime/src/fastlp.h
int ratio_test0(double *dy, int *idy,int ndy, double *y, double *ybar, double mu);
void fastlp(double *obj, double *mat, double *rhs, int *m0 , int *n0, double *opt, int *status, double *lam)
void solver20(int m,int n,int nz,int *ia, int *ka, double *a,double *b, double *c)
int ratio_test0(double *dy, int *idy,int ndy,double *y, double *ybar, double mu)
2) Next, I write the files for wrapping the C code in Cython:
%%file fastlp.pxd
cdef extern from "fastclime/src/fastlp.h":
void fastlp(double *obj, double *mat, double *rhs, int *m0 , int *n0, double *opt, int *status, double *lam)
%%file fastlp.pyx
cimport fastlp
def fastlp(double *obj, double *mat, double *rhs, int *m0 , int *n0, double *opt, int *status, double *lam):
fastlp.fastlp(*obj, *mat, *rhs, *m0, *n0, *opt, *status, *lam)
3) Next I use the distutils build system to compile.
%%file setup.py
from distutils.core import setup, Extension
from Cython.Build import cythonize
ext = Extension("fastlp",
sources=["fastlp.pyx", "fastclime/src/fastlp.c"])
setup(name="fastlp",
ext_modules = cythonize([ext]))
4) Finally I compile
! python setup.py build_ext -i
#4 is where things fail big time. Not sure how to resolve it in steps 1-3.
5) If all is successful (which it's not), I should be able to run this code (taken from the vignette)
import fastlp as flp
import numpy as np
A=np.array([-1.0,-1,0,1,-2,1]).reshape((3,2))
b=np.array([-1.0,-2,1])
c=np.array([-2.0,3])
flp.fastlp(c,A,b)
This should output the vector [2., 1.].
I'm not very experienced with Cython or Python for that matter, but getting this functionality would help out a lot with future projects. Could someone please let me know the proper way of taking a source file in C and getting through the steps in interfacing with numpy to create a Python module?
Thanks!
UPDATE:
I'm trying to follow the Numpy+Cython example on this site: https://scipy-lectures.github.io/advanced/interfacing_with_c/interfacing_with_c.html
I've since updated my code
%%cython
# import both numpy and the Cython declarations for numpy
import numpy as np
cimport numpy as np
# if you want to use the Numpy-C-API from Cython
# (not strictly necessary for this example)
np.import_array()
# cdefine the signature of our c function
cdef extern from "/home/bitnami/STA663-pura-project/fastclime/src/fastlp.h":
void fastlp(double *obj, double *mat, double *rhs, int *m0 , int *n0, double *opt, int *status, double *lam)
# create the wrapper code, with numpy type annotations
def fastlp_func(np.ndarray[double, ndim=1, mode="c"] obj not None,
np.ndarray[double, ndim=2, mode="c"] mat not None,
np.ndarray[double, ndim=1, mode="c"] rhs not None,
double lam):
cdef int m1 = mat.shape[0]
cdef int m2 = mat.shape[1]
cdef np.ndarray[double, ndim=1, mode='c'] z = np.zeros(obj.shape[0])
cdef double [:] zp = z #convert from Python to C object
cdef int stat = 0
cdef double lambd = 0
fastlp(<double*> np.PyArray_DATA(obj),
<double*> np.PyArray_DATA(mat.T),
<double*> np.PyArray_DATA(rhs),
&m1,
&m2,
&zp[0],
&stat,
&lambd
)
However, I get the following error (which I don't understand at all):
---------------------------------------------------------------------------
ImportError Traceback (most recent call last)
<ipython-input-85-29981393af41> in <module>()
----> 1 get_ipython().run_cell_magic(u'cython', u'', u'# import both numpy and the Cython declarations for numpy\nimport numpy as np\ncimport numpy as np\n\n# if you want to use the Numpy-C-API from Cython\n# (not strictly necessary for this example)\nnp.import_array()\n\n# cdefine the signature of our c function\ncdef extern from "/home/bitnami/STA663-pura-project/fastclime/src/fastlp.h":\n void fastlp(double *obj, double *mat, double *rhs, int *m0 , int *n0, double *opt, int *status, double *lam)\n \n# create the wrapper code, with numpy type annotations\ndef fastlp_func(np.ndarray[double, ndim=1, mode="c"] obj not None,\n np.ndarray[double, ndim=2, mode="c"] mat not None,\n np.ndarray[double, ndim=1, mode="c"] rhs not None,\n double lam):\n cdef int m1 = mat.shape[0]\n cdef int m2 = mat.shape[1]\n cdef np.ndarray[double, ndim=1, mode=\'c\'] z = np.zeros(obj.shape[0])\n cdef double [:] zp = z #convert from Python to C object\n cdef int stat = 0\n cdef double lambd = 0\n fastlp(<double*> np.PyArray_DATA(obj),\n <double*> np.PyArray_DATA(mat.T),\n <double*> np.PyArray_DATA(rhs),\n &m1,\n &m2,\n &zp[0], \n &stat, \n &lambd \n )')
/home/bitnami/anaconda/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc in run_cell_magic(self, magic_name, line, cell)
2160 magic_arg_s = self.var_expand(line, stack_depth)
2161 with self.builtin_trap:
-> 2162 result = fn(magic_arg_s, cell)
2163 return result
2164
/home/bitnami/anaconda/lib/python2.7/site-packages/IPython/extensions/cythonmagic.pyc in cython(self, line, cell)
/home/bitnami/anaconda/lib/python2.7/site-packages/IPython/core/magic.pyc in <lambda>(f, *a, **k)
191 # but it's overkill for just that one bit of state.
192 def magic_deco(arg):
--> 193 call = lambda f, *a, **k: f(*a, **k)
194
195 if callable(arg):
/home/bitnami/anaconda/lib/python2.7/site-packages/IPython/extensions/cythonmagic.pyc in cython(self, line, cell)
269 self._code_cache[key] = module_name
270
--> 271 module = imp.load_dynamic(module_name, module_path)
272 self._import_all(module)
273
ImportError: /home/bitnami/.cache/ipython/cython/_cython_magic_16feaf686ed172960f59fa6333ae74b5.so: undefined symbol: fastlp
How can I do this:
cdef class Tree:
cdef object key
cdef Tree left
cdef Tree right
cdef PyObject** find(self, key):
# get the address of self
# return &self
# return &(<PyObject*>self)
&self fails with Cannot take address of Python variable.
&(<PyObject*>self) fails with Taking address of non-lvalue, and I'm not sure that self is actually a PyObject*.
<void*>self and <PyObject*>self works just fine to get a pointer to self.
from ctypes import addressof, c_int
from cpython.ref cimport PyObject
from cython.operator import address
from libc.stdio cimport printf
cdef class A:
cdef object py
cdef int c
def __init__(self, py, c):
self.py = py
self.c = c
cdef void* addrvoid(self):
return <void*>self
cdef PyObject* addr(self):
return <PyObject*>self
cpdef run():
cdef A a
a = A([], 1)
# these are all equivalent
printf('a=%lu\n', <void*>a)
printf('a=%lu\n', <PyObject*>a)
printf('a=%lu\n', a.addrvoid())
printf('a=%lu\n', a.addr())
# type casting doesnt work with the extension's c attributes because it
# will translate to the arrow operator, like: (void *)__pyx_v_a->b)
# printf('%lu\n', <void*>a.c)
# printf('%lu\n', <void*>(a.c))
# printf('%lu\n', <PyObject*>(a.c))
# and address() dont work with python attributes
# printf('a.py=%lu\n', <void*>address(a.py))
# but address works with c attributes
printf('a.c=%lu\n', address(a.c))
# and type casting works with python attributes
printf('a.py=%lu\n', <void*>(a.py))
# it works with ctypes too
i = c_int(1)
print('cint=' + str(id(i)))
printf('cint=%lu\n', <void*>i)
# but it evaluates to the address of the python object
print('cint=' + str(addressof(i)))
Running this code will result in something like:
a=140516369271496
a=140516369271496
a=140516369271496
a=140516369271496
a.c=140516369271528
a.py=140516452410632
cint=140516465032184
cint=140516465032184
cint=140516465032264