I have a function, that should do report, if test function success.
But, I don't want to do report, if there is an Exception inside test function.
I try to use pytest.fixture, pytest.yield_fixture, but all of them always call finalizers. How can I understand, that Exception had been raised in test function?
test.py StatisticClass: start
FStatisticClass: stop
finalizer
contest of test.py:
#pytest.mark.usefixtures("statistic_maker")
def test_dummy():
raise Exception()
content of conftest.py:
class StatisticClass():
def __init__(self, req):
self.req = req
pass
def start(self):
print "StatisticClass: start"
def stop(self):
print "StatisticClass: stop"
def if_not_exception(self):
"""
I don't want to call this if Exception inside yield.
Maybe, there is any info in request object?
"""
print "finalizer"
#pytest.yield_fixture(scope="function")
def statistic_maker(request):
ds = StatisticClass(request)
ds.start()
request.addfinalizer(ds.if_not_exception)
yield
ds.stop()
P.S. I can't use decorator because, I use fixture.
Related
In a testing suite I have a fixture that drop all the tables in an engine, then start fresh and create all the tables. After this fixture logic, my test case runs, using the newly created table.
The fixture and the test case are run in the MainThread, while the database consumer is a web application server run in another thread.
However, I keep getting: sqlite3.OperationalError: no such table: ***
I've checked that they are using the same in-memory engine, but different connections(this is correct). And I've checked that the fixture does run before the consumer thread starts running.
What could be possible cause?
My code is as below:
import os
import pytest
import cherrypy
class DAL:
def __init__(self,
path="database",
filename=None,
conn_string=None,
echo=False):
if filename is None and conn_string is None:
conn_string = "sqlite:///:memory:"
elif conn_string is not None:
conn_string = conn_string
else:
conn_string = f'sqlite:///{os.path.abspath(path)}/{filename}'
self.conn_string = conn_string
engine = create_engine(conn_string, echo=echo)
Session_Factory = sessionmaker(bind=engine)
self.Session = sqlalchemy.orm.scoped_session(Session_Factory)
def __str__(self):
return f"<DAL>object: {self.conn_string}, at {hex(id(self))}"
def get_a_dbsession(self):
opened_db = self.Session()
return opened_db
def __enter__(self):
return self.get_a_dbsession()
def __exit__(self, exception_type, exception_value, exception_traceback):
opened_db = self.Session()
try:
opened_db.commit()
except:
opened_db.rollback()
raise
finally:
self.Session.remove()
else:
opened_db.close()
raise exception_type
def create_schema(self):
SchemaBase.metadata.create_all(self.Session().connection().engine)
class SAEnginePlugin(cherrypy.process.plugins.SimplePlugin):
def __init__(self, bus, dal):
"""
The plugin is registered to the CherryPy engine.
"""
cherrypy.process.plugins.SimplePlugin.__init__(self, bus)
self.dal = dal
def start(self):
self.bus.subscribe("bind-session", self.bind)
def stop(self):
self.bus.unsubscribe("bind-session", self.bind)
if self.dal:
del self.dal
def bind(self):
"""
Whenever this plugin receives the 'bind-session' message, it applies
this method and bind the received session to the engine.
"""
# self.dal.Session.configure(bind=self.dal.engine)
session = self.dal.get_a_dbsession()
return session
class SATool(cherrypy.Tool):
def __init__(self):
"""
This tool binds a session to the engine each time
a requests starts and commits/rollbacks whenever
the request terminates.
"""
cherrypy.Tool.__init__(self,
'on_start_resource',
self.bind_session,
priority=20)
def _setup(self):
cherrypy.Tool._setup(self)
cherrypy.request.hooks.attach('on_end_resource',
self.close_session,
priority=80)
def bind_session(self):
"""
Attaches a session to the request's scope by requesting
the SA plugin to bind a session to the SA engine.
"""
session = cherrypy.engine.publish('bind-session').pop()
cherrypy.request.db = session
def close_session(self):
"""
Commits the current transaction or rollbacks if an error occurs.
In all cases, the current session is unbound and therefore
not usable any longer.
"""
if not hasattr(cherrypy.request, 'db'):
return
try:
cherrypy.request.db.commit()
except:
cherrypy.request.db.rollback()
raise
finally:
cherrypy.request.db.close()
cherrypy.request.db = None
# Register the SQLAlchemy tool
cherrypy.tools.db = SATool()
class UnitServer:
...
#cherrypy.expose
#cherrypy.tools.json_in()
def list_filtered_entries(self):
...
queryOBJ = cherrypy.request.db.query(classmodel_obj)
...
############# main module code below ############:
# mocking 'db':
dal = database.DAL()
# configure cherrypy:
SAEnginePlugin(cherrypy.engine, dal).subscribe()
#pytest.fixture(autouse=True) # automatically run before every test case
def mocked_dal(request):
# first, clean the database by dropping all tables:
database.SchemaBase.metadata.drop_all(dal.Session().connection().engine)
# second, create the schema from blank:
dal.create_schema()
# third, insert some dummy data record:
...
db.commit()
class TestMyUnitServer(cherrypy.test.helper.CPWebCase):
#staticmethod
def setup_server():
...
server_app = UnitServer()
cherrypy.tree.mount(server_app, '', {'/': {'tools.db.on': True}})
def test_list_filtered_entries_allentries(self):
...
self.getPage('/list_filtered_entries',
headers=[("Accept", "application/json"),
('Content-type', 'application/json'),
('Content-Length',
str(len(json.dumps(query_params)))),
("Connection", "keep-alive"),
("Cache-Control", "max-age=0")],
body=serialized_query_params,
method="POST")
self.assertStatus('200 OK')
Trying to implement a context manager for a MySql connection I get an error message.
I have use MySql.connector module including the connection option to connect a database, and the pyMySql module, always getting the same result.
import pymysql
class MySQLConnector:
def __init__(self, con_dict):
self.cnx = None
self.con_dict = con_dict
def __enter__(self):
self.cnx = pymysql.connect(**self.con_dict)
return self.cnx
def __exit__(self):
self.cnx.close()
class ReceiveBroke(QDialog):
def __init__(self, db, config):
super().__init__()
with MySQLConnector(config) as
cur = self.cnx.cnx.cursor()
qry = "SELECT * FROM horses
cur.execute(qry)
result = cur.fetchall()
print(result)
self.setUI()
self.conTest()
def conTest(self):
if self.cnx.ope
print("y")
I hope to obtain a working context manager closing the database connection a finishing the with Block.
Result: Error Message. Always after executing the last line within the "with" block :"TypeError exit takes one positional argument but four were given" at which time the program crashes.
hope you found the error already. If not, I think the error message is quite clear:
TypeError exit takes one positional argument but four were given
__exit__ function needs 4 arguments: self, type, value, traceback. These last 3 relate to any exception that may happen in the with block.
I know I should specify argtypes for my C/C++ functions since some of my calls would otherwise result in stack corruption.
myCfunc.argtypes = [ct.c_void_p, ct.POINTER(ct.c_void_p)]
myCfunc.errcheck = my_error_check
In fact, I would like to verify that I did not forget to specify function prototypes (argtypes/errcheck) for any of my about 100 function calls...
Right now I just grep through my Python files and visually compare against my file containing the prototype definitions.
Is there a better way to verify that I have defined argtypes/errcheck for all my calls?
The mention of namespaces by #eryksun made me wrap the dll in a class that only exposes the explicitly annotated functions. As long as the dll doesn't have the function names "annotate" or "_error_check" (which my didn't), the following approach seems to work for me:
import ctypes as ct
class MyWinDll:
def __init__(self, dll_filename):
self._dll = ct.WinDLL(dll_filename)
# Specify function prototypes using the annotate function
self.annotate(self._dll.myCfunc, [ct.POINTER(ct.c_void_p)], self._error_check)
self.annotate(self._dll.myCfunc2, [ct.c_void_p], self._error_check)
...
def annotate(self, function, argtypes, errcheck):
# note that "annotate" may not be used as a function name in the dll...
function.argtypes = argtypes
function.errcheck = errcheck
setattr(self, function.__name__, function)
def _error_check(self, result, func, arguments):
if result != 0:
raise Exception
if __name__ == '__main__':
dll = MyWinDll('myWinDll.dll')
handle = ct.c_void_p(None)
# Now call the dll functions using the wrapper object
dll.myCfunc(ct.byref(handle))
dll.myCfunc2(handle)
Update: Comments by #eryksun made me try to improve the code by giving the user control of the WinDLL constructor and attempting to reduce repeated code:
import ctypes as ct
DEFAULT = object()
def annotate(dll_object, function_name, argtypes, restype=DEFAULT, errcheck=DEFAULT):
function = getattr(dll_object._dll, function_name)
function.argtypes = argtypes
# restype and errcheck is optional in the function_prototypes list
if restype is DEFAULT:
restype = dll_object.default_restype
function.restype = restype
if errcheck is DEFAULT:
errcheck = dll_object.default_errcheck
function.errcheck = errcheck
setattr(dll_object, function_name, function)
class MyDll:
def __init__(self, ct_dll, **function_prototypes):
self._dll = ct_dll
for name, prototype in function_prototypes.items():
annotate(self, name, *prototype)
class OneDll(MyDll):
def __init__(self, ct_dll):
# set default values for function_prototypes
self.default_restype = ct.c_int
self.default_errcheck = self._error_check
function_prototypes = {
'myCfunc': [[ct.POINTER(ct.c_void_p)]],
'myCfunc2': [[ct.c_void_p]],
# ...
'myCgetErrTxt': [[ct.c_int, ct.c_char_p, ct.c_size_t], DEFAULT, None]
}
super().__init__(ct_dll, **function_prototypes)
# My error check function actually calls the dll, so I keep it here...
def _error_check(self, result, func, arguments):
msg = ct.create_string_buffer(255)
if result != 0:
raise Exception(self.myCgetErrTxt(result, msg, ct.sizeof(msg)))
if __name__ == '__main__':
ct_dll = ct.WinDLL('myWinDll.dll')
dll = OneDll(ct_dll)
handle = ct.c_void_p(None)
dll.myCfunc(ct.byref(handle))
dll.myCfunc2(handle)
(I don't know if original code should be deleted, I kept it for reference.)
Here's a dummy class that can replace the DLL object's function call with a simple check to see the attributes have been defined:
class DummyFuncPtr(object):
restype = False
argtypes = False
errcheck = False
def __call__(self, *args, **kwargs):
assert self.restype
assert self.argtypes
assert self.errcheck
def __init__(self, *args):
pass
def __setattr__(self, key, value):
super(DummyFuncPtr, self).__setattr__(key, True)
To use it replace your DLL object's _FuncPtr class and then call each function to run the check, e.g.:
dll = ctypes.cdll.LoadLibrary(r'path/to/dll')
# replace the DLL's function pointer
# comment out this line to disable the dummy class
dll._FuncPtr = DummyFuncPtr
some_func = dll.someFunc
some_func.restype = None
some_func.argtypes = None
some_func.errcheck = None
another_func = dll.anotherFunc
another_func.restype = None
another_func.argtypes = None
some_func() # no error
another_func() # Assertion error due to errcheck not defined
The dummy class completely prevents the function from ever being called of course, so just comment out the replacement line to switch back to normal operation.
Note that it will only check each function when that function is called, so this would best be in a unit test file somewhere where the function is guaranteed to be called.
In Django I often assert the number of queries that should be made so that unit tests catch new N+1 query problems
from django import db
from django.conf import settings
settings.DEBUG=True
class SendData(TestCase):
def test_send(self):
db.connection.queries = []
event = Events.objects.all()[1:]
s = str(event) # QuerySet is lazy, force retrieval
self.assertEquals(len(db.connection.queries), 2)
In in SQLAlchemy tracing to STDOUT is enabled by setting the echo flag on
engine
engine.echo=True
What is the best way to write tests that count the number of queries made by SQLAlchemy?
class SendData(TestCase):
def test_send(self):
event = session.query(Events).first()
s = str(event)
self.assertEquals( ... , 2)
I've created a context manager class for this purpose:
class DBStatementCounter(object):
"""
Use as a context manager to count the number of execute()'s performed
against the given sqlalchemy connection.
Usage:
with DBStatementCounter(conn) as ctr:
conn.execute("SELECT 1")
conn.execute("SELECT 1")
assert ctr.get_count() == 2
"""
def __init__(self, conn):
self.conn = conn
self.count = 0
# Will have to rely on this since sqlalchemy 0.8 does not support
# removing event listeners
self.do_count = False
sqlalchemy.event.listen(conn, 'after_execute', self.callback)
def __enter__(self):
self.do_count = True
return self
def __exit__(self, *_):
self.do_count = False
def get_count(self):
return self.count
def callback(self, *_):
if self.do_count:
self.count += 1
Use SQLAlchemy Core Events to log/track queries executed (you can attach it from your unit tests so they don't impact your performance on the actual application:
event.listen(engine, "before_cursor_execute", catch_queries)
Now you write the function catch_queries, where the way depends on how you test. For example, you could define this function in your test statement:
def test_something(self):
stmts = []
def catch_queries(conn, cursor, statement, ...):
stmts.append(statement)
# Now attach it as a listener and work with the collected events after running your test
The above method is just an inspiration. For extended cases you'd probably like to have a global cache of events that you empty after each test. The reason is that prior to 0.9 (current dev) there is no API to remove event listeners. Thus make one global listener that accesses a global list.
what about the approach of using flask_sqlalchemy.get_debug_queries() btw. this is the methodology used by internal of Flask Debug Toolbar check its source
from flask_sqlalchemy import get_debug_queries
def test_list_with_assuring_queries_count(app, client):
with app.app_context():
# here generating some test data
for _ in range(10):
notebook = create_test_scheduled_notebook_based_on_notebook_file(
db.session, owner='testing_user',
schedule={"kind": SCHEDULE_FREQUENCY_DAILY}
)
for _ in range(100):
create_test_scheduled_notebook_run(db.session, notebook_id=notebook.id)
with app.app_context():
# after resetting the context call actual view we want asserNumOfQueries
client.get(url_for('notebooks.personal_notebooks'))
assert len(get_debug_queries()) == 3
keep in mind that for having reset context and count you have to call with app.app_context() before the exact stuff you want to measure.
Slightly modified version of #omar-tarabai's solution that removes the event listener when exiting the context:
from sqlalchemy import event
class QueryCounter(object):
"""Context manager to count SQLALchemy queries."""
def __init__(self, connection):
self.connection = connection.engine
self.count = 0
def __enter__(self):
event.listen(self.connection, "before_cursor_execute", self.callback)
return self
def __exit__(self, *args, **kwargs):
event.remove(self.connection, "before_cursor_execute", self.callback)
def callback(self, *args, **kwargs):
self.count += 1
Usage:
with QueryCounter(session.connection()) as counter:
session.query(XXX).all()
session.query(YYY).all()
print(counter.count) # 2
My WSGI application uses SQLAlchemy. I want to start session when request starts, commit it if it's dirty and request processing finished successfully, make rollback otherwise. So, I need to implement behavior of Django's TransactionMiddleware.
So, I suppose that I should create WSGI middleware and make following stuff:
Create and add DB session to environ on pre-processing.
Get DB session from environ and call commit() on post-processing, if no errors occurred.
Get DB session from environ and call rollback() on post-processing, if some errors occurred.
Step 1 is obvious for me:
class DbSessionMiddleware:
def __init__(self, app):
self.app = app
def __call__(self, environ, start_response):
environ['db_session'] = create_session()
return self.app(environ, start_response)
Step 2 and 3 - not. I found the example of post-processing task:
class Caseless:
def __init__(self, app):
self.app = app
def __call__(self, environ, start_response):
for chunk in self.app(environ, start_response):
yield chunk.lower()
It contains comment:
Note that the __call__ function is a Python generator, which is typical for this sort of “post-processing” task.
Could you please clarify how does it work and how can I solve my issue similarly.
Thanks,
Boris.
For step 1 I use SQLAlchemy scoped sessions:
engine = create_engine(settings.DB_URL, echo=settings.DEBUG, client_encoding='utf8')
Base = declarative_base()
sm = sessionmaker(bind=engine)
get_session = scoped_session(sm)
They return the same thread-local session for each get_session() call.
Step 2 and 3 for now is following:
class DbSessionMiddleware:
def __init__(self, app):
self.app = app
def __call__(self, environ, start_response):
try:
db.get_session().begin_nested()
return self.app(environ, start_response)
except BaseException:
db.get_session().rollback()
raise
finally:
db.get_session().commit()
As you can see, I start nested transaction on session to be able to rollback even queries that were already committed in views.