In SQLAlchemy, how should I specify that the relationship field is required? - sqlalchemy

I have a model that depends on some fields on another model. This fields should be present when the record is created, but I do not see a way to enforce that on the database:
class Study(db.Model):
id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
type = db.Column(Enum(StudyTypeChoices), nullable=False)
owner_id = db.Column(UUID(as_uuid=True), db.ForeignKey('owner.id'), nullable=False)
participants = db.relationship('Participant', lazy=True, cascade='save-update, merge, delete')
How can I make sure that 'participants' is provided when the Study record gets created (similar to what happens with the 'type' field)? I know I can put a wrapper around it to make sure of that, but I am wondering is there is a more neat way of doing it with sqlalchemy.
Edit: This is the definition of the Participant model
class Participant(UserBase):
id = db.Column(UUID(as_uuid=True), db.ForeignKey("user_base.id"), primary_key=True)
study_id = db.Column(UUID(as_uuid=True), db.ForeignKey('study.id'))

You can listen to before_flush events and prevent flushes containing studies without participants by raising an exception for instance.
#event.listens_for(Session, "before_flush")
def before_flush(session, flush_context, instances):
for instance in session.new: # might want to inspect session.dirty as well
if isinstance(instance, Study) and (
instance.participants is None or instance.participants == []
):
raise ValueError(
f"Study {instance} cannot have {instance.participants} participants."
)
This only checks for new studies, you might want to check in session.dirty as well for updated studies.
Full demo:
from sqlalchemy import Column, ForeignKey, Integer, create_engine, event
from sqlalchemy.orm import Session, declarative_base, relationship
Base = declarative_base()
class Study(Base):
__tablename__ = "study"
id = Column(Integer, primary_key=True)
participants = relationship("Participant", uselist=True, back_populates="study")
class Participant(Base):
__tablename__ = "participant"
id = Column(Integer, primary_key=True)
study_id = Column(Integer, ForeignKey("study.id"), nullable=True)
study = relationship("Study", back_populates="participants")
#event.listens_for(Session, "before_flush")
def before_flush(session, flush_context, instances):
for instance in session.new: # might want to inspect session.dirty as well
if isinstance(instance, Study) and (
instance.participants is None or instance.participants == []
):
raise ValueError(
f"Study {instance} cannot have {instance.participants} participants."
)
engine = create_engine("sqlite://", future=True, echo=True)
Base.metadata.create_all(engine)
s1 = Study()
p1_1 = Participant()
p1_2 = Participant()
s1.participants.extend([p1_1, p1_2])
s2 = Study()
with Session(bind=engine) as session:
session.add(s1)
session.commit() # OK
with Session(bind=engine) as session:
session.add(s2)
session.commit() # ValueError

Related

SQLAlchemy - Pytest - One or more mappers failed to initialize - can't proceed with initialization of other mappers

I am trying to create a DB with user and project tables having a many-to-many relation. A user can be a part of many projects and a project can have many users. I created the models for the user, project and the association tables and am able to create the database using alembic without any issues.
However, when i try to use the models. it throws an exception
sqlalchemy.exc.InvalidRequestError: One or more mappers failed to initialize - can't proceed with initialization of other mappers. Triggering mapper: 'mapped class User->users'. Original exception was: reverse_property 'user' on relationship User.projects references relationship ProjectUsers.user, which does not reference mapper mapped class User->users
Here is the code for the models
from app.modules.database import BaseModel
from sqlalchemy import Column, Integer, String, ForeignKey, JSON
from sqlalchemy.orm import relationship
from sqlalchemy_json import mutable_json_type
class Project(BaseModel):
__tablename__ = "projects"
__table_args__ = {'extend_existing': True}
project_name = Column(String, unique=True)
project_desc = Column(String)
users = relationship('app.modules.projects.project_models.ProjectUsers', back_populates='project')
owner_id = Column(Integer, ForeignKey('users.id'))
class ProjectUsers(BaseModel):
__tablename__ = "project_users"
__table_args__ = {'extend_existing': True}
user_id = Column(Integer, ForeignKey('users.id'), primary_key=True)
project_id = Column(Integer, ForeignKey('projects.id'), primary_key=True)
user = relationship("app.modules.users.user_models.User", back_populates="projects")
project = relationship("app.modules.projects.project_models.Project", back_populates="users")
The User model is :
from app.modules.database import BaseModel
from sqlalchemy import Column, Integer, String, ForeignKey
from sqlalchemy.orm import relationship
from dataclasses import dataclass
#dataclass
class User(BaseModel):
__tablename__ = "users"
__table_args__ = {'extend_existing': True}
email = Column(String, unique=True)
username = Column(String, unique=True)
f_name = Column(String)
l_name = Column(String)
m_name = Column(String)
role_id = Column(Integer, ForeignKey('roles.id'))
#user_role = relationship('app.modules.roles.role_models.Role', backref="users")
# A user can have access to many projects where they are not the owners of the project.
#projects = relationship('app.modules.projects.project_models.Project', secondary="project_users", back_populates="users")
projects = relationship('app.modules.projects.project_models.ProjectUsers', back_populates='user')
def __repr__(self):
return '<User: {}>'.format(self.username)
Alembic creates the corresponding tables with the proper associations without any issues, but running a simple test like creating a new User() or Project() throws the above exception.
I think the issue might come from using a dataclass without using the correct mapping method. There are three ways to apply ORM Mappings to a dataclass. You seem to use declarative mapping so I'll just apply the declarative mapping method for SQLAlchemy 1.4.
#mapper_registry.mapped
#dataclass
class User:
__tablename__ = "users"
__table_args__ = {'extend_existing': True}
__sa_dataclass_metadata_key__ = "sa"
email: str | None = field(init=False, metadata={"sa": Column(String, unique=True)})
username: str | None = field(init=False, metadata={"sa": Column(String, unique=True)})
f_name: str | None = field(init=False, metadata={"sa": Column(String)})
l_name: str | None = field(init=False, metadata={"sa": Column(String)})
m_name: str | None = field(init=False, metadata={"sa": Column(String)})
role_id: int | None = field(init=False, metadata={"sa": Column(Integer, ForeignKey('roles.id'))})
#user_role = relationship('app.modules.roles.role_models.Role', backref="users")
# A user can have access to many projects where they are not the owners of the project.
#projects = relationship('app.modules.projects.project_models.Project', secondary="project_users", back_populates="users")
projects: list[Project] = field(default_factory=list, metadata={"sa": relationship('app.modules.projects.project_models.ProjectUsers', back_populates='user')})
# define repr by using the dataclasses.field repr kwarg
If that alone does not solve your problem, please add the definition of app.modules.database.BaseModel to your question.

sqlalchemy relationship select from other table instead of insert

I'm having difficulties in relationships. I have users and roles and defined model and schema for them.
the problem is when I try to add a new user with a previously defined role (I have its ID and name)it will try to update/insert the role table by the value the user specifies. but I only want to select from roles and specify that as a user role and not updating the role table(if role not found return error).
what I want to achieve is how to limit SQLalchemy in updating related tables by the value that the user specifies.
here is my models:
class User(db.Model):
"""user model
"""
__tablename__ = 'user'
id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, unique=True, nullable=False)
username = db.Column(db.String(40), unique=True, nullable=False)
password = db.Column(db.String(255), nullable=False)
role_id = db.Column(UUID(as_uuid=True), db.ForeignKey('role.id') , nullable=False)
class Role(db.Model):
"""role model
"""
__tablename__ = 'role'
id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, unique=True, nullable=False)
name = db.Column(db.String(40), unique=True, nullable=False)
perm_add = db.Column(db.Boolean, default=False)
perm_edit = db.Column(db.Boolean, default=False)
perm_del = db.Column(db.Boolean, default=False)
here is the schema that I defined:
class UserSchema(ma.SQLAlchemyAutoSchema):
password = ma.String(load_only=True, required=True)
email = ma.String(required=True)
role = fields.Nested("RoleSchema", only=("id", "name"), required=True)
class Meta:
model = User
sqla_session = db.session
load_instance = True
and I grab user input which is checked by schema and commit it to DB.
schema = UserSchema()
user = schema.load(request.json)
db.session.add(user)
try:
db.session.commit()
the point is here I could not change anything regarding role name or ID as it seems it is changed by schema even before applying to DB (I mean request.json)
In my example, I am using the additional webargs library. It facilitates validation on the server side and enables clean notation. Since marschmallow is based on webargs anyway, I think the addition makes sense.
I have based myself on your specifications. Depending on what you intend to do further, you may need to make adjustments.
I added a relationship to the user model to make the role easier to use.
class User(db.Model):
"""user model"""
# ...
# The role is mapped by sqlalchemy using the foreign key
# as an object and can be reached via a virtual relationship.
role = db.relationship('Role')
I have allowed the foreign key as a query parameter in the schema and limited the nested schema to the output. The email is assigned to the username.
class RoleSchema(ma.SQLAlchemyAutoSchema):
class Meta:
model = Role
load_instance = True
class UserSchema(ma.SQLAlchemyAutoSchema):
# The entry of the email is converted into a username.
username = ma.String(required=True, data_key='email')
password = ma.String(required=True, load_only=True)
# The foreign key is only used here for loading.
role_id = ma.Integer(required=True, load_only=True)
# The role is dumped with a query.
role = ma.Nested("RoleSchema", only=("id", "name"), dump_only=True)
class Meta:
model = User
load_instance = True
include_relationships = True
It is now possible to query the role from the database and react if it does not exist. The database table for the roles is no longer updated automatically.
from flask import abort
from sqlalchemy.exc import SQLAlchemyError
from webargs.flaskparser import use_args, use_kwargs
# A schema-compliant input is expected as JSON
# and passed as a parameter to the function.
#blueprint.route('/users/', methods=['POST'])
#use_args(UserSchema(), location='json')
def user_new(user):
# The role is queried from the database and assigned to the user object.
# If not available, 404 Not Found is returned.
user_role = Role.query.get_or_404(user.role_id)
user.role = user_role
# Now the user can be added to the database.
db.session.add(user)
try:
db.session.commit()
except SQLAlchemyError as exc:
# If an error occurs while adding to the database,
# 422 Unprocessable Entity is returned
db.session.rollback()
abort(422)
# Upon successful completion, the new user is returned
# with the status code 201 Created.
user_schema = UserSchema()
user_data = user_schema.dump(user)
return jsonify(data=user_data), 201

Dialect-specific table and column names in sqlalchemy while reusing an existing ORM mapping

I use sqlalchemy's ORM library to interface with one of our application's databases. On one host, the database backend of an application instance is PostgreSQL, while on another, we use MySQL for historical reasons.
The schemas (and relationships) are identical on both backends, but the table and column names in PostgreSQL are lowercase, while on MySQL they are uppercase. Is there any way I can adapt my schema in sqlalchemy such that I don't need to duplicate code?
The following is an example of my schema. Hint, the application is a Confluence wiki ;)
from sqlalchemy import (
BigInteger,
Column,
DateTime,
String,
)
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class Space(Base):
__tablename__ = 'spaces'
spaceid = Column(BigInteger, primary_key=True)
spacename = Column(String(255))
spacekey = Column(String(255), nullable=False, unique=True)
lowerspacekey = Column(String(255), nullable=False, index=True)
spacedescid = Column(BigInteger, index=True)
homepage = Column(BigInteger, index=True)
creator = Column(String(255), index=True)
creationdate = Column(DateTime, index=True)
lastmodifier = Column(String(255), index=True)
lastmoddate = Column(DateTime)
spacetype = Column(String(255))
spacestatus = Column(String(255), index=True)
Thanks to #rfkortekaas' suggestion, I managed to solve the issue by automapping the schema and subsequently overriding the naming scheme for table and column names.
The documentation of sqlalchemy has two very useful pages:
Overriding Naming Schemes
Intercepting Column Definitions
For posterity's sake:
from sqlalchemy import event
from sqlalchemy.ext.automap import automap_base
from .utilities import to_pascal_case, to_snake_case
AutomapBase = automap_base()
#event.listens_for(AutomapBase.metadata, "column_reflect")
def column_reflect(inspector, table, column_info):
column_info['key'] = to_snake_case(column_info['name'])
def classname_for_table(base, table_name, table):
return to_pascal_case(table_name)
if __name__ == '__main__':
dbcs = 'postgresql://username:password#127.0.0.1:3306/database'
automap_engine = create_engine(dbcs, future=True, echo=verbose > 2)
AutomapBase.prepare(
autoload_with=automap_engine,
classname_for_table=classname_for_table,
)
# Access the automapped classes with AutomapBase.classes.<class_name>

How to build backref with both associatition object and secondaryjoin?

I need some models for instance following:
Work - e.g. works of literature.
Worker - e.g. composer, translator or something similar has contribution to work.
Thus, a 'type' field is required to distinguish workers by division of work. As SQLAlchemy's documentation, this case can benifit from association object like following:
class Work(base):
id = Column(Integer, primary_key=True)
name = Column(String(50))
description = Column(Text)
class Worker(base):
id = Column(Integer, primary_key=True)
name = Column(String(50))
description = Column(Text)
class Assignment(base):
work_id = Column(Integer, Foreignkey('work.id'), primary_key=True)
worker_id = Column(Integer, Foreignkey('worker.id'), primary_key=True)
type = Column(SmallInteger, nullable=True)
Nonetheless, how to take advantage of backref and alternatvie join condition for building relation immediately to implement that each Work object can retrieve and modify corresponding Worker(s) via different attributions for distinction. For example:
work = session.query(Work).get(1)
work.name
>>> 'A Dream of The Red Mansions'
work.composers
>>> [<Worker('Xueqin Cao')>]
work.translators
>>> [<Worker('Xianyi Yang')>, <Worker('Naidie Dai')>]
Vice versa:
worker = session.query(Worker).get(1)
worker.name
>>> 'Xueqin Cao'
worker.composed
>>> [<Work('A Dream of The Red Mansions')>]
worker.translated
>>> []
Adding secondaryjoin directly without secondary specified seems not feasible, besides, SQLAlchemy's docs notes that:
When using the association object pattern, it is advisable that the association-mapped table not be used as the secondary argument on a relationship() elsewhere, unless that relationship() contains the option viewonly=True. SQLAlchemy otherwise may attempt to emit redundant INSERT and DELETE statements on the same table, if similar state is detected on the related attribute as well as the associated object.
Then, is there some way to build these relations elegantly and readily ?
There's three general ways to go here.
One is, do a "vanilla" setup where you have "work"/"workers" set up without distinguishing on "type" - then, use relationship() for "composer", "composed", "translator", "translated" by using "secondary" to Assignment.__table__ along with custom join conditions, as well as viewonly=True. So you'd do writes via the vanilla properties only. A disadvantage here is that there's no immediate synchronization between the "vanilla" and "specific" collections.
Another is, same with the "vanilla" setup, but just use plain Python descriptors to give "composer", "composed", "translator", "translated" views in memory, that is, [obj.worker for obj in self.workers if obj.type == 'composer']. This is the simplest way to go. Whatever you put in the "vanilla" collections shows right up in the "filtered" collection, the SQL is simple, and there's fewer SELECT statements in play (one per Worker/Work instead of N per Worker/Work).
Finally, the approach that's closest to what you're asking, with primary joins and backrefs, but note with the association object, the backrefs are between Work/Assignment and Assignment/Worker, but not between Work/Worker directly. This approach probably winds up using more SQL to get at the results but is the most complete, and also has the nifty feature that the "type" is written automatically. We're also using a "one way backref", as Assignment doesn't have a simple way of relating back outwards (there's ways to do it but it would be tedious). Using a Python function to automate creation of the relationships reduces the boilerplate, and note here I'm using a string for "type", this can be an integer if you add more arguments to the system:
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.associationproxy import association_proxy
Base = declarative_base()
def _work_assignment(name):
assign_ = relationship("Assignment",
primaryjoin="and_(Assignment.work_id==Work.id, "
"Assignment.type=='%s')" % name,
back_populates="work", cascade="all, delete-orphan")
assoc = association_proxy("%s_assign" % name, "worker",
creator=lambda worker: Assignment(worker=worker, type=name))
return assoc, assign_
def _worker_assignment(name):
assign_ = relationship("Assignment",
primaryjoin="and_(Assignment.worker_id==Worker.id, "
"Assignment.type=='%s')" % name,
back_populates="worker", cascade="all, delete-orphan")
assoc = association_proxy("%s_assign" % name, "work",
creator=lambda work: Assignment(work=work, type=name))
return assoc, assign_
class Work(Base):
__tablename__ = 'work'
id = Column(Integer, primary_key=True)
name = Column(String(50))
description = Column(Text)
composers, composer_assign = _work_assignment("composer")
translators, translator_assign = _work_assignment("translator")
class Worker(Base):
__tablename__ = 'worker'
id = Column(Integer, primary_key=True)
name = Column(String(50))
description = Column(Text)
composed, composer_assign = _worker_assignment("composer")
translated, translator_assign = _worker_assignment("translator")
class Assignment(Base):
__tablename__ = 'assignment'
work_id = Column(Integer, ForeignKey('work.id'), primary_key=True)
worker_id = Column(Integer, ForeignKey('worker.id'), primary_key=True)
type = Column(String, nullable=False)
worker = relationship("Worker")
work = relationship("Work")
e = create_engine("sqlite://", echo=True)
Base.metadata.create_all(e)
session = Session(e)
ww1, ww2, ww3 = Worker(name='Xueqin Cao'), Worker(name='Xianyi Yang'), Worker(name='Naidie Dai')
w1 = Work(name='A Dream of The Red Mansions')
w1.composers.append(ww1)
w1.translators.extend([ww2, ww3])
session.add(w1)
session.commit()
work = session.query(Work).get(1)
assert work.name == 'A Dream of The Red Mansions'
assert work.composers == [ww1]
assert work.translators == [ww2, ww3]
worker = session.query(Worker).get(ww1.id)
assert worker.name == 'Xueqin Cao'
assert worker.composed == [work]
assert worker.translated == []
worker.composed[:] = []
# either do this...
session.expire(work, ['composer_assign'])
# or this....basically need composer_assign to reload
# session.commit()
assert work.composers == []

sqlalchemy multiple databases with same table names not working

I have two databases that I'm working with in with Python using SQLAlchemy, the databases share table names and therefore I'm getting an error message when running the code.
The error message is :
sqlalchemy.exc.InvalidRequestError: Table 'wo' is already defined for this MetaData instance. Specify 'extend_existing=True' to redefine options and columns on an existing Table object.
The simplified code is below:
from sqlalchemy import create_engine, Column, Integer, String, DateTime, ForeignKey
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker, relationship, backref
from mysql.connector.connection import MySQLConnection
Base = declarative_base()
def get_characterset_info(self):
return self.get_charset()
MySQLConnection.get_characterset_info = MySQLConnection.get_charset
mysqlengine = create_engine('mysql+mysqlconnector://......../mp2', echo=True)
MYSQLSession = sessionmaker(bind=mysqlengine)
mysqlsession= MYSQLSession()
MP2engine = create_engine('mssql+pyodbc://......../mp2', echo=True)
MP2Session = sessionmaker(bind=MP2engine)
mp2session= MP2Session()
class MYSQLWo(Base):
__tablename__= 'wo'
wonum = Column(String, primary_key=True)
taskdesc = Column(String)
comments = relationship("MYSQLWocom", order_by="MYSQLWocom.wonum", backref='wo')
class MYSQLWocom (Base):
__tablename__='wocom'
wonum = Column(String, ForeignKey('wo.wonum'), primary_key=True)
comments = Column(String, primary_key=True)
class MP2Wo(Base):
__tablename__= 'wo'
wonum = Column(String, primary_key=True)
taskdesc = Column(String)
comments = relationship("MP2Wocom", order_by="MP2Wocom.wonum", backref='wo')
class MP2Wocom (Base):
__tablename__='woc'
wonum = Column(String, ForeignKey('wo.wonum'), primary_key=True)
location = Column(String)
sublocation1 = Column(String)
texts = Column(String, primary_key=True)
How do I deal with databases having the same table structure? I'm guessing it has something to do with the MetaData instance, but the SQLAlchemy documentation gets a little confusing when talking about the difference in the class declarative and classical usage..
Since in reality the tables had different structures, the solution was to simply create a separate declarative base. If the tables indeed had the same structure, I would have only needed one class for both tables.
Base = declarative_base()
Base2 = declarative_base() #this is all I needed
class MYSQLWo(Base):
....
class MYSQLWocom(Base):
....
class MP2Wo(Base2):
....
class MP2Wocom(Base2)
http://groups.google.com/group/sqlalchemy/browse_thread/thread/afe09d6387a4dc69?hl=en
You can use one db instance with two Model to bypass this problem.
And this can also be used to implement a master/slave use case in Flask-SQLAlchemy.
Just like this:
app = Flask(__name__)
app.config['SQLALCHEMY_BINDS'] = {'rw': 'rw', 'r': 'r'}
db = SQLAlchemy(app)
db.Model_RW = db.make_declarative_base()
class A(db.Model):
__tablename__ = 'common'
class B(db.Model_RW):
__tablename__ = 'common'