I am using a synchronous (SQLite) session in a FastAPI Dependency as shown in the SQLModel Tutorial:
def get_session():
with Session(engine) as session:
yield session
However, when a route and a sub-dependency "depends" on get_session, the session is created twice resulting in errors such as:
sqlalchemy.exc.InvalidRequestError: Object '<User at 0x7f218cbf6a00>' is already attached to session '1' (this is '2')
Here is a reduced example:
from fastapi import Depends
from sqlmodel import Session, create_engine, select
def get_session():
with Session(engine) as session:
yield session
async def get_current_user(token: str = Depends(oauth2_scheme), session: Session = Depends(get_session)):
# Token read omitted 👈
user = session.get(User, token_data.user_uuid)
if user is None:
raise credentials_exception
return user
#app.patch("/me", response_model=UserRead)
def update_current_user(
user: UserUpdate,
current_user: User = Depends(get_current_user).
session: Session = Depends(get_session),
):
user_data = user.dict(exclude_unset=True)
for key, value in user_data.items():
setattr(current_user, key, value)
# Session and session from current_user are different ⚠️
session.add(current_user)
session.commit()
session.refresh(current_user)
return current_user
How can this be prevented? I would expect a Dependency to be reused as long as use_cache=True.
Related
In a testing suite I have a fixture that drop all the tables in an engine, then start fresh and create all the tables. After this fixture logic, my test case runs, using the newly created table.
The fixture and the test case are run in the MainThread, while the database consumer is a web application server run in another thread.
However, I keep getting: sqlite3.OperationalError: no such table: ***
I've checked that they are using the same in-memory engine, but different connections(this is correct). And I've checked that the fixture does run before the consumer thread starts running.
What could be possible cause?
My code is as below:
import os
import pytest
import cherrypy
class DAL:
def __init__(self,
path="database",
filename=None,
conn_string=None,
echo=False):
if filename is None and conn_string is None:
conn_string = "sqlite:///:memory:"
elif conn_string is not None:
conn_string = conn_string
else:
conn_string = f'sqlite:///{os.path.abspath(path)}/{filename}'
self.conn_string = conn_string
engine = create_engine(conn_string, echo=echo)
Session_Factory = sessionmaker(bind=engine)
self.Session = sqlalchemy.orm.scoped_session(Session_Factory)
def __str__(self):
return f"<DAL>object: {self.conn_string}, at {hex(id(self))}"
def get_a_dbsession(self):
opened_db = self.Session()
return opened_db
def __enter__(self):
return self.get_a_dbsession()
def __exit__(self, exception_type, exception_value, exception_traceback):
opened_db = self.Session()
try:
opened_db.commit()
except:
opened_db.rollback()
raise
finally:
self.Session.remove()
else:
opened_db.close()
raise exception_type
def create_schema(self):
SchemaBase.metadata.create_all(self.Session().connection().engine)
class SAEnginePlugin(cherrypy.process.plugins.SimplePlugin):
def __init__(self, bus, dal):
"""
The plugin is registered to the CherryPy engine.
"""
cherrypy.process.plugins.SimplePlugin.__init__(self, bus)
self.dal = dal
def start(self):
self.bus.subscribe("bind-session", self.bind)
def stop(self):
self.bus.unsubscribe("bind-session", self.bind)
if self.dal:
del self.dal
def bind(self):
"""
Whenever this plugin receives the 'bind-session' message, it applies
this method and bind the received session to the engine.
"""
# self.dal.Session.configure(bind=self.dal.engine)
session = self.dal.get_a_dbsession()
return session
class SATool(cherrypy.Tool):
def __init__(self):
"""
This tool binds a session to the engine each time
a requests starts and commits/rollbacks whenever
the request terminates.
"""
cherrypy.Tool.__init__(self,
'on_start_resource',
self.bind_session,
priority=20)
def _setup(self):
cherrypy.Tool._setup(self)
cherrypy.request.hooks.attach('on_end_resource',
self.close_session,
priority=80)
def bind_session(self):
"""
Attaches a session to the request's scope by requesting
the SA plugin to bind a session to the SA engine.
"""
session = cherrypy.engine.publish('bind-session').pop()
cherrypy.request.db = session
def close_session(self):
"""
Commits the current transaction or rollbacks if an error occurs.
In all cases, the current session is unbound and therefore
not usable any longer.
"""
if not hasattr(cherrypy.request, 'db'):
return
try:
cherrypy.request.db.commit()
except:
cherrypy.request.db.rollback()
raise
finally:
cherrypy.request.db.close()
cherrypy.request.db = None
# Register the SQLAlchemy tool
cherrypy.tools.db = SATool()
class UnitServer:
...
#cherrypy.expose
#cherrypy.tools.json_in()
def list_filtered_entries(self):
...
queryOBJ = cherrypy.request.db.query(classmodel_obj)
...
############# main module code below ############:
# mocking 'db':
dal = database.DAL()
# configure cherrypy:
SAEnginePlugin(cherrypy.engine, dal).subscribe()
#pytest.fixture(autouse=True) # automatically run before every test case
def mocked_dal(request):
# first, clean the database by dropping all tables:
database.SchemaBase.metadata.drop_all(dal.Session().connection().engine)
# second, create the schema from blank:
dal.create_schema()
# third, insert some dummy data record:
...
db.commit()
class TestMyUnitServer(cherrypy.test.helper.CPWebCase):
#staticmethod
def setup_server():
...
server_app = UnitServer()
cherrypy.tree.mount(server_app, '', {'/': {'tools.db.on': True}})
def test_list_filtered_entries_allentries(self):
...
self.getPage('/list_filtered_entries',
headers=[("Accept", "application/json"),
('Content-type', 'application/json'),
('Content-Length',
str(len(json.dumps(query_params)))),
("Connection", "keep-alive"),
("Cache-Control", "max-age=0")],
body=serialized_query_params,
method="POST")
self.assertStatus('200 OK')
I'm just trying to get FastAPI unittests working with SQLAlchemy, but I'm having trouble testing with objects created in the database. I have the following setup, as per the docs.
main.py
routes = [
APIRoute('/games/', views.games_list, name='index', response_class=HTMLResponse),
]
settings = Settings()
app = FastAPI(debug=settings.debug, routes=routes)
views.py
# Dependency
def get_db():
db = SessionLocal()
try:
yield db
finally:
db.close()
async def games_list(request: Request, db: Session = Depends(get_db)):
settings = Settings()
games = db.query(Game).all()
return settings.templates.TemplateResponse('games/list.jinja', {'request': request, 'games': games})
database.py
def prepare_database(settings):
engine = create_engine(settings.database_url)
Base.metadata.create_all(engine)
return engine
engine = prepare_database(delete_existing=False, settings=Settings())
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
conftest.py
#pytest.fixture
def db_conn():
session = SessionLocalTesting()
try:
yield session
finally:
session.close()
#pytest.fixture
def cli(db_conn):
def override_get_db():
session = SessionLocalTesting()
try:
yield session
finally:
session.close()
app.dependency_overrides[get_db] = override_get_db
with TestClient(app) as client:
yield client
test file
def test_games_list(cli, factory, db_conn):
factory.create_game()
# This will return the game I have created with my factory, definitely in the test db.
print(db_conn.query(Game.name).all())
r = cli.get('/games/')
assert 'DnD Game' in r.content.decode()
My issue is that I can't get the objects from the test db in view. If I print(db.bind.database.url ) in views the test database is not being used, so it's trying to get items from the real database, not the test one.
So it looks like get_db is not getting overridden, though I'm not sure why.
Thanks in advance
I think you forgot to specify the scope of the fixture when you should call a fixture
Create a DB connection and override DB dependency through #pytest.fixture there is a kind of scope available in pytest.fixture(scope='session') there module level as well and other lots off scope available in Type of scope
I have mentioned a link which helps you Use test DB intend of real
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from sqlalchemy_utils import create_database, drop_database
from app.main import app
from db.base import Base
from db.settings import get_database
from settings.config import (
TEST_DATABASE_URI as SQLALCHEMY_DATABASE_URL,
)
# SQLALCHEMY_DATABASE_URL = "sqlite:///./test.db"
engine = create_engine(SQLALCHEMY_DATABASE_URL, pool_pre_ping=True)
def override_get_db():
"""" Override """
try:
db = Session(autocommit=False, autoflush=False, bind=engine)
yield db
finally:
db.close()
#pytest.fixture(scope="session", autouse=True)
def create_db():
""" creating db model in database """
create_database(SQLALCHEMY_DATABASE_URL)
print("\n" + "\x1b[6;30;42m" + "Creating test database." + "\x1b[0m")
Base.metadata.create_all(bind=engine)
app.dependency_overrides[get_database] = override_get_db
yield 1
drop_database(SQLALCHEMY_DATABASE_URL)
print("\n" + "\x1b[6;30;42m" + "Delete test database." + "\x1b[0m")
#pytest.fixture()
def get_db_session():
""" Getting session for db transaction """
session = Session(autocommit=False, autoflush=False, bind=engine)
yield session
session.close()
#pytest.fixture()
def client():
""" Getting testclient of app """
with TestClient(app) as client:
yield client
If you still face the problem or not let me know
I am creating a test Flask API, and have created a Database class that I use from my main app. I am using pymysql to access my MySQL DB but I am having trouble figuring out when to close the cursor and connection. Right now I have
import pymysql
class Database:
def __init__(self):
host = '127.0.0.1'
user = 'root'
password = ''
db = 'API'
self.con = pymysql.connect(host=host, user=user, password=password, db=db, cursorclass=pymysql.cursors.DictCursor, autocommit=True)
self.cur = self.con.cursor()
def getUser(self, id):
sql = 'SELECT * from users where id = %d'
self.cur.execute(sql, (id))
result = self.cur.fetchall()
return result
def getAllUsers(self):
sql = 'SELECT * from users'
self.cur.execute(sql)
result = self.cur.fetchall()
return result
def AddUser(self, firstName, lastName, email):
sql = "INSERT INTO `users` (`firstName`, `lastName`, `email`) VALUES (%s, %s, %s)"
self.cur.execute(sql, (firstName, lastName, email))
I have tried adding self.cur.close() and self.con.close() after each execution of the cursor in the functions but then I get an error the next time I call a function saying the cursor is closed, or after I do an insert statement it won't show the new value even though it was inserted correctly into MySQL. How do I know when to close the cursor, and how to start it back up properly with each call to a method?
This sounds like a great use case for a python context manager. Context Managers allow you to properly manage resources, such as a database connection, by allowing you to specify how your resource's set-up and tear down methods should work. You can create your own custom context manager in one of two ways: First, by wrapping your database class, and implementing the required methods for the context manager: __init__(), __enter__(), and __exit__(). Second, by utilizing a #contextmanager decorator on a function definition and creating a generator for your database resource within said function definition. I will show both approaches and let you decide which one is your preference. The __init__() method is the initialization method for your custom context manager, similar to the initialization method used for custom python classes. The __enter__() method is your setup code for your custom context manager. Lastly, the __exit()__ method is your teardown code for your custom context manager. Both approaches utilize these methods with the main difference being the first method will explicitly state these methods within your class definition. Where as in the second approach, all the code up to your generator's yield statement is your initialization and setup code and all the code after the yield statement is your teardown code. I would also consider extracting out your user based database actions into a user model class as well. Something along the lines of:
custom context manager: (class based approach):
import pymysql
class MyDatabase():
def __init__(self):
self.host = '127.0.0.1'
self.user = 'root'
self.password = ''
self.db = 'API'
self.con = None
self.cur = None
def __enter__(self):
# connect to database
self.con = pymysql.connect(host=self.host, user=self.user, password=self.password, db=self.db, cursorclass=pymysql.cursors.DictCursor, autocommit=True)
self.cur = self.con.cursor()
return self.cur
def __exit__(self, exc_type, exc_val, traceback):
# params after self are for dealing with exceptions
self.con.close()
user.py (refactored):'
# import your custom context manager created from the step above
# if you called your custom context manager file my_database.py: from my_database import MyDatabase
import <custom_context_manager>
class User:
def getUser(self, id):
sql = 'SELECT * from users where id = %d'
with MyDatabase() as db:
db.execute(sql, (id))
result = db.fetchall()
return result
def getAllUsers(self):
sql = 'SELECT * from users'
with MyDatabase() as db:
db.execute(sql)
result = db.fetchall()
return result
def AddUser(self, firstName, lastName, email):
sql = "INSERT INTO `users` (`firstName`, `lastName`, `email`) VALUES (%s, %s, %s)"
with MyDatabase() as db:
db.execute(sql, (firstName, lastName, email))
context manager (decorator approach):
from contextlib import contextmanager
import pymysql
#contextmanager
def my_database():
try:
host = '127.0.0.1'
user = 'root'
password = ''
db = 'API'
con = pymysql.connect(host=host, user=user, password=password, db=db, cursorclass=pymysql.cursors.DictCursor, autocommit=True)
cur = con.cursor()
yield cur
finally:
con.close()
Then within your User class you could use the context manager by first importing the file and then using it similar to as before:
with my_database() as db:
sql = <whatever sql stmt you wish to execute>
#db action
db.execute(sql)
Hopefully that helps!
Recently I've got a SQLAlchemy InvalidRequestError.
The error log shows:
InvalidRequestError: Transaction <sqlalchemy.orm.session.SessionTransaction object at
0x106830dd0> is not on the active transaction list
In what circumstance this error will be raised???
-----Edit----
# the following two line actually in my decorator
s = Session()
s.add(model1)
# refer to <http://techspot.zzzeek.org/2012/01/11/django-style-database-routers-in-sqlalchemy/>
s2 = Session().using_bind('master')
model2 = s2.query(Model2).with_lockmode('update').get(1)
model2.somecolumn = 'new'
s2.commit()
This exception is raised
-----Edit2 -----
s = Session().using_bind('master')
model = Model(user_id=123456)
s.add(model)
s.flush()
# here, raise the exception.
# I add log in get_bind() of RoutingSession. when doing 'flush', the _name is None, and it returns engines['slave'].
#If I use commit() instead of flush(), then it commits successfully
I change the using_bind method as the following and it works well.
def using_bind(self, name):
self._name = name
return self
The previous RoutingSession:
class RoutingSession(Session):
_name = None
def get_bind(self, mapper=None, clause=None):
logger.info(self._name)
if self._name:
return engines[self._name]
elif self._flushing:
logger.info('master')
return engines['master']
else:
logger.info('slave')
return engines['slave']
def using_bind(self, name):
s = RoutingSession()
vars(s).update(vars(self))
s._name = name
return s
that's an internal assertion which should never occur. There's no way to answer this question without at least a full stack trace, if perhaps you are improperly using the Session in a concurrent fashion, or manipulating its internals. I can only show that exception raised if I manipulate private methods or state pertaining to the Session object.
Here's that:
from sqlalchemy.orm import Session
s = Session()
s2 = Session()
t = s.transaction
t2 = s2.transaction
s2.transaction = t # nonsensical assignment of the SessionTransaction
# from one Session to also be referred to by another,
# corrupts the transaction chain by leaving out "t2".
# ".transaction" should never be assigned to on the outside
t2.rollback() # triggers the assertion case
basically, the above should never happen, since you're not supposed to assign to ".transaction". that's a read-only attribute.
My WSGI application uses SQLAlchemy. I want to start session when request starts, commit it if it's dirty and request processing finished successfully, make rollback otherwise. So, I need to implement behavior of Django's TransactionMiddleware.
So, I suppose that I should create WSGI middleware and make following stuff:
Create and add DB session to environ on pre-processing.
Get DB session from environ and call commit() on post-processing, if no errors occurred.
Get DB session from environ and call rollback() on post-processing, if some errors occurred.
Step 1 is obvious for me:
class DbSessionMiddleware:
def __init__(self, app):
self.app = app
def __call__(self, environ, start_response):
environ['db_session'] = create_session()
return self.app(environ, start_response)
Step 2 and 3 - not. I found the example of post-processing task:
class Caseless:
def __init__(self, app):
self.app = app
def __call__(self, environ, start_response):
for chunk in self.app(environ, start_response):
yield chunk.lower()
It contains comment:
Note that the __call__ function is a Python generator, which is typical for this sort of “post-processing” task.
Could you please clarify how does it work and how can I solve my issue similarly.
Thanks,
Boris.
For step 1 I use SQLAlchemy scoped sessions:
engine = create_engine(settings.DB_URL, echo=settings.DEBUG, client_encoding='utf8')
Base = declarative_base()
sm = sessionmaker(bind=engine)
get_session = scoped_session(sm)
They return the same thread-local session for each get_session() call.
Step 2 and 3 for now is following:
class DbSessionMiddleware:
def __init__(self, app):
self.app = app
def __call__(self, environ, start_response):
try:
db.get_session().begin_nested()
return self.app(environ, start_response)
except BaseException:
db.get_session().rollback()
raise
finally:
db.get_session().commit()
As you can see, I start nested transaction on session to be able to rollback even queries that were already committed in views.