cython class with an attributes as python object instance - cython

Let's have an example.
cdef class Example:
attr2 = None
cdef attr3
cdef object attr4
def __init__(self):
self.attr1 = Obj()
self.attr2 = Obj()
self.attr3 = Obj()
self.attr4 = Obj()
The line with the assignment to self.attr1 will raise an AttributeError saying: "object has no attribute 'attr1'.
If try with the self.attr2 it raises also the exception but with the message: "object attribute 'attr2' is read-only".
And if uses the keyword cdef, an it doesn't have an explicit type, the compilation process will fail.
If this attribute is defined with the type object, it looks nice. But different instances of the Example class will have this attr4 like a singleton and any interaction with one of them will be visible for the other instances and, in this case, I want to have an Obj() instance unique for each of them.

You can try something like this :
class Obj(object):
# At first I define a class Obj() to match your example
def __init__(self):
self.abc = 32
self.xyz = "xyzxyz"
# Then I can define the Example class
cdef class Example:
cdef public:
object attr1, attr2, attr3, attr4
def __init__(self):
self.attr1 = Obj()
self.attr2 = Obj()
self.attr3 = Obj()
self.attr4 = Obj()
It seems to be cythonized/compiled without error and you have access to the attributes related to the Obj object :
In [11]:
a = Example()
a.attr1.abc
​
Out[11]:
32

Related

Problem with PettingZoo and Stable-Baselines3 with a ParallelEnv

I am having trouble in making things work with a Custom ParallelEnv I wrote by using PettingZoo. I am using SuperSuit's ss.pettingzoo_env_to_vec_env_v1(env) as a wrapper to Vectorize the environment and make it work with Stable-Baseline3 and documented here.
You can find attached a summary of the most relevant part of the code:
from typing import Optional
from gym import spaces
import random
import numpy as np
from pettingzoo import ParallelEnv
from pettingzoo.utils.conversions import parallel_wrapper_fn
import supersuit as ss
from gym.utils import EzPickle, seeding
def env(**kwargs):
env_ = parallel_env(**kwargs)
env_ = ss.pettingzoo_env_to_vec_env_v1(env_)
#env_ = ss.concat_vec_envs_v1(env_, 1)
return env_
petting_zoo = env
class parallel_env(ParallelEnv, EzPickle):
metadata = {'render_modes': ['ansi'], "name": "PlayerEnv-Multi-v0"}
def __init__(self, n_agents: int = 20, new_step_api: bool = True) -> None:
EzPickle.__init__(
self,
n_agents,
new_step_api
)
self._episode_ended = False
self.n_agents = n_agents
self.possible_agents = [
f"player_{idx}" for idx in range(n_agents)]
self.agents = self.possible_agents[:]
self.agent_name_mapping = dict(
zip(self.possible_agents, list(range(len(self.possible_agents))))
)
self.observation_spaces = spaces.Dict(
{agent: spaces.Box(shape=(len(self.agents),),
dtype=np.float64, low=0.0, high=1.0) for agent in self.possible_agents}
)
self.action_spaces = spaces.Dict(
{agent: spaces.Discrete(4) for agent in self.possible_agents}
)
self.current_step = 0
def seed(self, seed=None):
self.np_random, seed = seeding.np_random(seed)
def observation_space(self, agent):
return self.observation_spaces[agent]
def action_space(self, agent):
return self.action_spaces[agent]
def __calculate_observation(self, agent_id: int) -> np.ndarray:
return self.observation_space(agent_id).sample()
def __calculate_observations(self) -> np.ndarray:
observations = {
agent: self.__calculate_observation(
agent_id=agent)
for agent in self.agents
}
return observations
def observe(self, agent):
return self.__calculate_observation(agent_id=agent)
def step(self, actions):
if self._episode_ended:
return self.reset()
observations = self.__calculate_observations()
rewards = random.sample(range(100), self.n_agents)
self.current_step += 1
self._episode_ended = self.current_step >= 100
infos = {agent: {} for agent in self.agents}
dones = {agent: self._episode_ended for agent in self.agents}
rewards = {
self.agents[i]: rewards[i]
for i in range(len(self.agents))
}
if self._episode_ended:
self.agents = {} # To satisfy `set(par_env.agents) == live_agents`
return observations, rewards, dones, infos
def reset(self,
seed: Optional[int] = None,
return_info: bool = False,
options: Optional[dict] = None,):
self.agents = self.possible_agents[:]
self._episode_ended = False
self.current_step = 0
observations = self.__calculate_observations()
return observations
def render(self, mode="human"):
# TODO: IMPLEMENT
print("TO BE IMPLEMENTED")
def close(self):
pass
Unfortunately when I try to test with the following main procedure:
from stable_baselines3 import DQN, PPO
from stable_baselines3.common.env_checker import check_env
from dummy_env import dummy
from pettingzoo.test import parallel_api_test
if __name__ == '__main__':
# Testing the parallel algorithm alone
env_parallel = dummy.parallel_env()
parallel_api_test(env_parallel) # This works!
# Testing the environment with the wrapper
env = dummy.petting_zoo()
# ERROR: AssertionError: The observation returned by the `reset()` method does not match the given observation space
check_env(env)
# Model initialization
model = PPO("MlpPolicy", env, verbose=1)
# ERROR: ValueError: could not broadcast input array from shape (20,20) into shape (20,)
model.learn(total_timesteps=10_000)
I get the following error:
AssertionError: The observation returned by the `reset()` method does not match the given observation space
If I skip check_env() I get the following one:
ValueError: could not broadcast input array from shape (20,20) into shape (20,)
It seems like that ss.pettingzoo_env_to_vec_env_v1(env) is capable of splitting the parallel environment in multiple vectorized ones, but not for the reset() function.
Does anyone know how to fix this problem?
Plese find the Github Repository to reproduce the problem.
You should double check the reset() function in PettingZoo. It will return None instead of an observation like GYM
Thanks to discussion I had in the issue section of the SuperSuit repository, I am able to post the solution to the problem. Thanks to jjshoots!
First of all it is necessary to have the latest SuperSuit version. In order to get that I needed to install Stable-Baseline3 using the instructions here to make it work with gym 0.24+.
After that, taking the code in the question as example, it is necessary to substitute
def env(**kwargs):
env_ = parallel_env(**kwargs)
env_ = ss.pettingzoo_env_to_vec_env_v1(env_)
#env_ = ss.concat_vec_envs_v1(env_, 1)
return env_
with
def env(**kwargs):
env_ = parallel_env(**kwargs)
env_ = ss.pettingzoo_env_to_vec_env_v1(env_)
env_ = ss.concat_vec_envs_v1(env_, 1, base_class="stable_baselines3")
return env_
The outcomes are:
Outcome 1: leaving the line with check_env(env) I got an error AssertionError: Your environment must inherit from the gym.Env class cf https://github.com/openai/gym/blob/master/gym/core.py
Outcome 2: removing the line with check_env(env), the agent starts training successfully!
In the end, I think that the argument base_class="stable_baselines3" made the difference.
Only the small problem on check_env remains to be reported, but I think it can be considered as trivial if the training works.

Is there a way to ensure that all my ctypes have argtypes?

I know I should specify argtypes for my C/C++ functions since some of my calls would otherwise result in stack corruption.
myCfunc.argtypes = [ct.c_void_p, ct.POINTER(ct.c_void_p)]
myCfunc.errcheck = my_error_check
In fact, I would like to verify that I did not forget to specify function prototypes (argtypes/errcheck) for any of my about 100 function calls...
Right now I just grep through my Python files and visually compare against my file containing the prototype definitions.
Is there a better way to verify that I have defined argtypes/errcheck for all my calls?
The mention of namespaces by #eryksun made me wrap the dll in a class that only exposes the explicitly annotated functions. As long as the dll doesn't have the function names "annotate" or "_error_check" (which my didn't), the following approach seems to work for me:
import ctypes as ct
class MyWinDll:
def __init__(self, dll_filename):
self._dll = ct.WinDLL(dll_filename)
# Specify function prototypes using the annotate function
self.annotate(self._dll.myCfunc, [ct.POINTER(ct.c_void_p)], self._error_check)
self.annotate(self._dll.myCfunc2, [ct.c_void_p], self._error_check)
...
def annotate(self, function, argtypes, errcheck):
# note that "annotate" may not be used as a function name in the dll...
function.argtypes = argtypes
function.errcheck = errcheck
setattr(self, function.__name__, function)
def _error_check(self, result, func, arguments):
if result != 0:
raise Exception
if __name__ == '__main__':
dll = MyWinDll('myWinDll.dll')
handle = ct.c_void_p(None)
# Now call the dll functions using the wrapper object
dll.myCfunc(ct.byref(handle))
dll.myCfunc2(handle)
Update: Comments by #eryksun made me try to improve the code by giving the user control of the WinDLL constructor and attempting to reduce repeated code:
import ctypes as ct
DEFAULT = object()
def annotate(dll_object, function_name, argtypes, restype=DEFAULT, errcheck=DEFAULT):
function = getattr(dll_object._dll, function_name)
function.argtypes = argtypes
# restype and errcheck is optional in the function_prototypes list
if restype is DEFAULT:
restype = dll_object.default_restype
function.restype = restype
if errcheck is DEFAULT:
errcheck = dll_object.default_errcheck
function.errcheck = errcheck
setattr(dll_object, function_name, function)
class MyDll:
def __init__(self, ct_dll, **function_prototypes):
self._dll = ct_dll
for name, prototype in function_prototypes.items():
annotate(self, name, *prototype)
class OneDll(MyDll):
def __init__(self, ct_dll):
# set default values for function_prototypes
self.default_restype = ct.c_int
self.default_errcheck = self._error_check
function_prototypes = {
'myCfunc': [[ct.POINTER(ct.c_void_p)]],
'myCfunc2': [[ct.c_void_p]],
# ...
'myCgetErrTxt': [[ct.c_int, ct.c_char_p, ct.c_size_t], DEFAULT, None]
}
super().__init__(ct_dll, **function_prototypes)
# My error check function actually calls the dll, so I keep it here...
def _error_check(self, result, func, arguments):
msg = ct.create_string_buffer(255)
if result != 0:
raise Exception(self.myCgetErrTxt(result, msg, ct.sizeof(msg)))
if __name__ == '__main__':
ct_dll = ct.WinDLL('myWinDll.dll')
dll = OneDll(ct_dll)
handle = ct.c_void_p(None)
dll.myCfunc(ct.byref(handle))
dll.myCfunc2(handle)
(I don't know if original code should be deleted, I kept it for reference.)
Here's a dummy class that can replace the DLL object's function call with a simple check to see the attributes have been defined:
class DummyFuncPtr(object):
restype = False
argtypes = False
errcheck = False
def __call__(self, *args, **kwargs):
assert self.restype
assert self.argtypes
assert self.errcheck
def __init__(self, *args):
pass
def __setattr__(self, key, value):
super(DummyFuncPtr, self).__setattr__(key, True)
To use it replace your DLL object's _FuncPtr class and then call each function to run the check, e.g.:
dll = ctypes.cdll.LoadLibrary(r'path/to/dll')
# replace the DLL's function pointer
# comment out this line to disable the dummy class
dll._FuncPtr = DummyFuncPtr
some_func = dll.someFunc
some_func.restype = None
some_func.argtypes = None
some_func.errcheck = None
another_func = dll.anotherFunc
another_func.restype = None
another_func.argtypes = None
some_func() # no error
another_func() # Assertion error due to errcheck not defined
The dummy class completely prevents the function from ever being called of course, so just comment out the replacement line to switch back to normal operation.
Note that it will only check each function when that function is called, so this would best be in a unit test file somewhere where the function is guaranteed to be called.

SQLAlchemy: exclude a column from being pickled

Is it possible to mark a column as do-not-pickle in SQLAlchemy, such that it will get loaded from the database on demand after it is unpickled instead?
This can be partially achieved with a deferred column, but if the column is subsequently loaded it will also then get pickled.
(I'm asking as it looks like there's still a problem in Python2.7 with pickling Geoalchemy2 Geometry columns due to the unpickleable nature of the built in buffer: https://github.com/geoalchemy/geoalchemy/issues/24 )
This is a little bit involved because you can't just prevent pickling of a column, since what controls the deferred loading is actually in the _sa_instance_state attribute, so even if you don't pickle some attribute in your instance dictionary SQLAlchemy does not know that that means it's expired.
A (slightly hacky) solution is to expire the attribute when you unpickle:
class Foo(Base):
__tablename__ = "foo"
id = Column(Integer, primary_key=True)
foo = Column(String)
bar = Column(String)
def __getstate__(self):
return {k: v for k, v in self.__dict__.items() if k != "bar"}
def __setstate__(self, d):
for k, v in d.items():
self.__dict__[k] = v
self._sa_instance_state._expire_attributes(self.__dict__, ("bar",))
Note that the result of unpickling is a detached instance, so you need to be careful how you use it:
foo = session.query(Foo).first()
foo = pickle.loads(pickle.dumps(foo))
print(foo.foo) # fine
# print(foo.bar) # sqlalchemy.orm.exc.DetachedInstanceError
foo = session.merge(foo, load=False)
print(foo.foo) # no query
print(foo.bar) # causes a query
Here's a naive __getstate__ implementation that doesn't work, but will hopefully prompt the right answer:
Base = declarative_base(metadata=route_metadata)
cols_to_omit = ['my_col_1']
class MyClass(Base):
my_col_0 = Column(Integer)
my_col_1 = Column(Integer)
def __getstate__(self):
return dict((c, getattr(self, c)) for c in \
self.__table__.columns.keys() \
if c not in cols_to_omit)
def __setstate__(self, *args):
for c, v in args[0].items():
setattr(self, c, v)
from pickle import *
pp = dumps(MyClass.query.first())
obj = loads(pp)
This gives AttributeError: 'MyClass' object has no attribute '_sa_instance_state'

Bounded Function displaying address in Python

class NonVerticalLine:
def __init__(self, point_1, point_2):
self.point_1=point_1
self.point_2=point_2
def slope(self):
p1=self.point_1
p2=self.point_2
return (p2.y-p1.y)/(p2.x-p1.x)
When i pass the value to _init and access function slope it gives--->
>>> from quiz_5 import *
>>> p1 = Point(1,2)
>>> p2 = Point(4,4)
>>> line = NonVerticalLine(point_1 = p1, point_2 = p2)
>>> line.slope
<bound method NonVerticalLine.slope of <quiz_5.NonVerticalLine object at 0x105b00160>>
It should be 0.6666666666666666
In that case, try
class NonVerticalLine:
def __init__(self, point_1, point_2):
self.point_1=point_1
self.point_2=point_2
#property
def slope(self):
p1=self.point_1
p2=self.point_2
return (p2.y-p1.y)/(p2.x-p1.x)
The #property decorator is used to implement getters and setters in Python, so line.slope would be equivalent to line.slope() without the decorator. In this case it seems that this behaviour is what you want.

Python 3 Accessing variable result from another Class

I have a little problem with a variable update.
I have my variable declared in my first function as such self.TestVar = 0
then if a certain count ==2 self.TestVar = 2
in a second function (in the same class) but called from within another class I want returning self.TestVar. no way.
AttributeError: 'ThndClass' object has no attribute 'TestVar'
I am most certainly not doing the good way, all I want is accessing self.TestVar = 2 from my other class that's it's but I can't find a proper way to do so in Python.
It looks like my issue is that I get my self.TestVar = 2 in a "if" statement which make it live in another scope (or I might be wrong).
import sys
from PIL import Image
from PyQt4 import QtCore, QtGui
class MainWindow(QtGui.QWidget):
def __init__(self):
super(MainWindow, self).__init__()
self.initUI()
def initUI(self):
self.TestVar = 0
self.TheCount = 2
if self.TheCount ==2:
self.TestVar = 2
ThndClass()
def Getit(self):
print("called correctly")
print(self.TestVar)
return self.TestVar
def main():
app = QtGui.QApplication([])
mw = MainWindow()
sys.exit(app.exec_())
class ThndClass(QtGui.QWidget):
def __init__(self):
super(ThndClass, self).__init__()
self.initUI2()
def initUI2(self):
print("Class Called")
print(MainWindow.Getit(self))
if __name__ == '__main__':
main()
If I remove the 2nd Class call :
import sys
from PIL import Image
from PyQt4 import QtCore, QtGui
class MainWindow(QtGui.QWidget):
def __init__(self):
super(MainWindow, self).__init__()
self.initUI()
def initUI(self):
self.TestVar = 0
self.TheCount = 2
if self.TheCount ==2:
self.TestVar = 2
self.Getit()
def Getit(self):
print("called correctly")
print(self.TestVar)
return self.TestVar
def main():
app = QtGui.QApplication([])
mw = MainWindow()
sys.exit(app.exec_())
if __name__ == '__main__':
main()
This works correctly, but I want to be able to call def Getit() from another class and get my result. Or simply get a way to directly access self.TestVar from my other class.
When you call
MainWindow.Getit(self)
in ThndClass.initUI2, you are treating MainWindow and ThndClass interchangeably, when they do not have the same attributes. Here is an actual minimal example:
class Parent():
def __init__(self):
pass
class Child1(Parent):
def __init__(self):
super().__init__()
self.foo = "foo"
def method(self):
print(type(self))
print(self.foo)
class Child2(Parent):
def __init__(self):
super().__init__()
self.bar = "bar"
c1 = Child1()
Child1.method(c1) # pass Child1 instance to Child1 instance method
c2 = Child2()
Child1.method(c2) # pass Child2 instance to Child1 instance method
and full output:
<class '__main__.Child1'> # gets a Child1 instance
foo # first call succeeds
<class '__main__.Child2'> # gets a Child2 instance (which doesn't have 'foo')
Traceback (most recent call last):
File "C:/Python34/so.py", line 25, in <module>
Child1.method(c2)
File "C:/Python34/so.py", line 11, in method
print(self.foo)
AttributeError: 'Child2' object has no attribute 'foo' # second call fails
However, as it is not clear what exactly the code is supposed to be doing, I can't suggest a fix. I don't know why you create but don't assign a ThndClass instance in MainWindow.initUI, for example.
Here is one possible fix; pass a Child1 instance to Child2.__init__, then use it either as an argument to Child2.method:
class Child2(Parent):
def __init__(self, c1): # provide Child1 instance as parameter
super().__init__()
self.bar = "bar"
self.method(c1) # pass instance to Child2.method
def method(self, c1):
c1.method() # call Child1.method with c1 as self parameter
(Note that c1.method() is equivalent to Child1.method(c1).)
or make it an instance attribute:
class Child2(Parent):
def __init__(self, c1): # provide Child1 instance as parameter
super().__init__()
self.bar = "bar"
self.c1 = c1 # make Child1 instance a Child2 instance attribute
self.method() # now no argument needed
def method(self):
self.c1.method() # call Child1.method with c1 as self parameter
(Note that self.c1.method() is equivalent to Child1.method(self.c1).)
In use (either way):
>>> c1 = Child1()
>>> c2 = Child2(c1)
<class '__main__.Child1'> # Child1.method gets a Child1 instance
foo # and is called successfully
Thank's to your help jonrsharpe here's my working code :)
import sys
from PIL import Image
from PyQt4 import QtCore, QtGui
class MainWindow(QtGui.QWidget):
def __init__(self):
super(MainWindow, self).__init__()
self.initUI()
def initUI(self):
self.TestVar = 0
self.TheCount = 2
if self.TheCount ==2:
self.TestVar = 2
Themain = self
ThndClass(Themain)
def Getit(self):
print("called correctly")
print(self.TestVar)
return self.TestVar
def main():
app = QtGui.QApplication([])
mw = MainWindow()
sys.exit(app.exec_())
class ThndClass(QtGui.QWidget):
def __init__(self, Themain):
super(ThndClass, self).__init__()
self.Themain = Themain
self.initUI2()
def initUI2(self):
print("Class Called")
print(self.Themain.Getit())
if __name__ == '__main__':
main()
All working good now : ) Thanks you very much !