SQLAlchemy self referencial table get the number of grandchilds - sqlalchemy

I got a User model and each user has another user as a parent. Now, to get the number of children (users that belong to a given model instance) I have defined this property:
class User( object ):
#property
def childsCount( self ):
return object_session(self).scalar(
select([func.count(User.users_id)]).where(User.parent_id==self.users_id)
)
...which works ok. What I don't know is how do I get the count of grandchilds? Or even grandgrandchilds.
Any ideas?

Use aliased to composed the deeper level WHERE clauses. In fact you can do it somewhat more generic:
#property
def childrenCount(self):
return self.count_children(0)
#property
def grandchildrenCount(self):
return self.count_children(1)
#property
def grandgrandchildrenCount(self):
return self.count_children(2)
def count_children(self, level=0):
a = [aliased(User) for _ in range(level + 1)]
qry = select([func.count(a[0].users_id)]).where(a[-1].parent_id==self.users_id)
# insert all the intermediate JOINs
for _i in range(level):
qry = qry.where(a[_i].parent_id == a[_i+1].users_id)
return Session.object_session(self).scalar(qry)
Although it looks somewhat cryptic, what it really does is expaned as below (add one alias and where clause for each deeper level):
#property
def children1Count(self):
a0 = aliased(User)
qry = select([func.count(a0.users_id)]).where(a0.parent_id==self.users_id)
return Session.object_session(self).scalar(qry)
#property
def children2Count(self):
a0 = aliased(User)
a1 = aliased(User)
qry = select([func.count(a0.users_id)]).where(a0.parent_id==a1.users_id).where(a1.parent_id==self.users_id)
return Session.object_session(self).scalar(qry)
#property
def children3Count(self):
a0 = aliased(User)
a1 = aliased(User)
a2 = aliased(User)
qry = select([func.count(a0.users_id)]).where(a0.parent_id==a1.users_id).where(a1.parent_id==a2.users_id).where(a2.parent_id==self.users_id)
return Session.object_session(self).scalar(qry)
For first level only you can actually have a nicer query with with_parent:
#property
def childrenCount(self):
return Session.object_session(self).query(User).with_parent(self).count()

Related

how make a conditions in django annotate?

i'm, trying to do something like this :
c = if x > y return x else return y
inner annotate function
class factura (models.MODEL):
price = Model.integerField(max_length=50, null=False)
articles = Models.charField(Max_length=50, default=0, null=False)
iva = Models.integerField(max_length=50)
discount = Model.integerField((max_length=50)
factura.objects.annotate(
total = if total_articles > price return iva else return
thnks
You can define a model method to compare the value:
class factura (models.Model):
...
#property
def check_article(self):
if int(self.articles) > int(self.price): return 'iva'
else: return
You can then do:
qs=factura.objects.all()
for instance in qs:
print(instance.check_article)

Pattern for rest query params in flask

Is there a pattern for dealing with query params in a flask rest server? I know I can create a sql query word for word using string manipulation in python, but I find that to be ugly and error prone, I was wondering if there is a better way. Here's what I have:
param1 = request.args.get('param1', type = int)
param2 = request.args.get('param2', type = int)
if param1 is not None:
if param2 is not None:
cursor.execute("SELECT * FROM table WHERE p1 = %s AND p2 = %s", (str(param1), str(param2)))
else:
cursor.execute("SELECT * FROM table WHERE p1 = %s", (str(param1),))
else:
if param2 is not None:
cursor.execute("SELECT * FROM table WHERE p2 = %s", (str(param2),))
else:
cursor.execute("SELECT * FROM table")
It's easy to see the number of possible SQL statements is 2 to the number of parameters, which grows out of control... so, again, without using string manipulation to custom build the sql query, is there an idiom or pattern that is used to accomplish this in a more elegant way? Thanks.
Loop through your parameters.
params = []
for i in range(1, HoweverManyParamsYouNeed):
params.append(request.args.get('param' + str(i), type = int))
s = ""
for i in range(1, len(params)):
if params[ i ] is not None:
if not s:
s = "p" + str(i) + " = " + str(params[ i ])
else:
s = s + " AND p" + str(i) + " = " + str(params[ i ])
full = "SELECT * FROM table"
if s:
full = full + " WHERE " + s
cursor.execute(full)
You might need to correct this code, since I do not have a way to run it.
I suggest using ORM(https://en.wikipedia.org/wiki/Object-relational_mapping) instead of raw sql queries.
First you need install flask-sqlalchemy (https://flask-sqlalchemy.palletsprojects.com/)
Then define your model
class MyModel(db.Model):
id = db.Column(db.Integer, primary_key=True)
column1 = db.Column(db.Integer)
column2 = db.Column(db.Integer)
Let's say you have your filter lookup somewhere
allowed_filters = {"column1", "column2"}
Finally instead of cursor you can use SQLAlchemy's ORM to retrieve your filtered objects.
query = MyModel.query
for field, value in request.args.items():
if field in allowed_filters:
query = query.filter(getattr(MyModel, field) == value)
my_object_list = list(query.all())
If you really want to create your queries manually you can always iterate over args:
where_clause = ""
params = []
for field, value in request.args.items():
if field in allowed_filters:
if len(where_clause) > 0:
where_clause += " AND "
where_clause += "{} = %s".format(field)
params.append(value)
if len(where_clause) > 0:
cursor.execute("SELECT * FROM table WHERE {}".format(where_clause), tuple(params))
else:
cursor.execute("SELECT * FROM table")

How do I add an aggregate as "virtual column" in a result?

I've got a query that normally looks like
def get_models_with_children(ids):
query = MyModel.query.filter(MyModel.id.in_(ids))
.join(Child, Child.parent_id = Child.id)
.groupBy(MyModel.id)
.having(func.count(Child.id) > 0)
return query.all()
Sometimes, I want to actually retrieve the count, as well. I can make that happen easily enough:
def get_models_with_children(ids, return_count):
query = MyModel.query
if return_count:
query = query.add_columns(func.count(Child.id).label("child_count"))
query = query.filter(MyModel.id.in_(ids))
.join(Child, Child.parent_id = Child.id)
.groupBy(MyModel.id)
.having(func.count(Child.id) > 0)
return query.all()
This works fine, but now, instead of a List[MyModel] coming back, I've got a differently shaped result with MyModel and child_count keys. If I want the MyModel's id, I do result[0].id if I didn't add the count, and result[0].MyModel.id if I did.
Is there any way I can flatten the result, so that the thing that's returned looks like a MyModel with an extra child_count column?
def do_stuff_with_models():
result = get_models_with_children([1, 2, 3], True)
for r in result:
# can't do this, but I want to:
print(r.id)
print(r.child_count)
# instead I have to do this:
print(r.MyModel.id)
print(r.child_count)
sqlalchemy.util.KeyedTuple is the type * of differently shaped result with MyModel and child_count keys:
Result rows returned by Query that contain multiple
ORM entities and/or column expressions make use of this
class to return rows.
You can effectively flatten them by explictly specifying the columns for your query. Here follows a complete example (tested on SQLAlchemy==1.3.12).
Plain table column attribute
Models:
import sqlalchemy as sa
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class User(Base):
__tablename__ = 'user'
user_id = sa.Column(sa.Integer, sa.Sequence('user_id_seq'), primary_key=True)
username = sa.Column(sa.String(80), unique=True, nullable=False)
def __repr__(self):
return f'User({self.user_id!r}, {self.username!r})'
class Token(Base):
__tablename__ = 'token'
token_id = sa.Column(sa.Integer, sa.Sequence('token_id_seq'), primary_key=True)
user_id = sa.Column(sa.Integer, sa.ForeignKey('user.user_id'), nullable=False)
user = sa.orm.relationship('User')
value = sa.Column(sa.String(120), nullable=False)
def __repr__(self):
return f'Token({self.user.username!r}, {self.value!r})'
Connect and fill some data:
engine = sa.create_engine('sqlite://')
Base.metadata.create_all(engine)
Session = sa.orm.sessionmaker(bind=engine)
session = Session()
user1 = User(username='joe')
user2 = User(username='john')
token1 = Token(user=user1, value='q1w2e3r4t56')
session.add_all([user1, user2, token1])
session.commit()
Now, let's define the "virtual" column as whether user has a token:
query = session.query(User)
exists = (
sa.exists()
.where(User.user_id == Token.user_id)
.correlate(User)
.label("has_token")
)
query = query.add_columns(exists)
query.all() # [(User(1, 'joe'), True), (User(2, 'john'), False)]
It's the undesired shape. And here's how to flatten it:
query = session.query(*[getattr(User, n) for n in User.__table__.columns.keys()])
query = query.add_columns(exists)
query.all() # [(1, 'joe', True), (2, 'john', False)]
It's all possible to define columns for an existing query, given that you know the model:
query = session.query(User)
# later down the line
query = query.with_entities(*[
getattr(User, n) for n in User.__table__.columns.keys()])
query = query.add_columns(exists)
query.all() # [(1, 'joe', True), (2, 'john', False)]
Column bundle
The same can be achieved with sqlalchemy.orm.Bundle and passing single_entity to it.
bundle = sa.orm.Bundle(
'UserBundle', User.user_id, User.username, exists, single_entity=True)
query = session.query(bundle)
query.all() # [(1, 'joe', True), (2, 'john', False)]
Issue with relationship attribute
With complex models it gets complicated. It's possible to inspect the model (mapped class) attributes with sqlalchemy.orm.mapper.Mapper.attrs and take class_attribute:
# replace
[getattr(User, n) for n in User.__table__.columns.keys()]
# with
[mp.class_attribute for mp in sa.inspect(User).attrs]
But in this case relationship attributes turn into their target tables in FROM clause of the query without ON clause, effectively producing a cartesian product. And the "joins" have to be defined manually, so it's not a good solution. See this answer and a SQLAlchemy user group discussion.
Query expression attribute
Myself I ended up using query expressions, because of the issues with relationships in existing code. It's possible to get away with minimal modification of the model, with query-time SQL expressions as mapped attributes.
User.has_tokens = sa.orm.query_expression()
...
query = query.options(sa.orm.with_expression(User.has_tokens, exists))
query.all() # [User(1, 'joe'), User(2, 'john')]
[u.has_tokens for u in query.all()] # [True, False]
* Actually it's generated on-the-fly sqlalchemy.util._collections.result with MRO of sqlalchemy.util._collections.result, sqlalchemy.util._collections._LW, class sqlalchemy.util._collections.AbstractKeyedTuple, tuple, object, but that's details. More details on how the class is created with lightweight_named_tuple are available in this answer.

Sqlalchemy eager loading of parent all properties in joined table inheritance

I have the following problem:
I have a hierachy of classes with joined table inheritance:
class AdGroupModel(Base, AdwordsRequestMixin):
__tablename__ = 'ad_groups'
db_id = Column(BigInteger, primary_key=True)
created_at = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now())
# ----RELATIONS-----
# campaign MANY-to-ONE
campaign_db_id = Column(BigInteger,
ForeignKey('campaigns.db_id', ondelete='CASCADE'),
nullable = True,
)
# # ads ONE-to-MANY
ads = relationship("AdModel",
backref="ad_group",
lazy="subquery",
passive_deletes=True,
single_parent=True,
cascade="all, delete, delete-orphan")
# # # keywords ONE-to-MANY
criteria = relationship("AdGroupCriterionModel",
backref="ad_group",
lazy="subquery",
passive_deletes=True,
single_parent=True,
cascade="all, delete, delete-orphan")
# Joined Table Inheritance
type = Column(Unicode(50))
__mapper_args__ = {
'polymorphic_identity': 'ad_group',
'polymorphic_on': type
}
class AdGroupCriterionModel(Base, AdGroupDependenceMixin):
__tablename__ = 'ad_group_criterion'
db_id = Column(BigInteger, primary_key=True)
destination_url = Column(Unicode, nullable=True)
status = Column(Enum("PAUSED", "ACTIVE", "DELETED",
name='criterion_status'), default="ACTIVE")
# ----RELATIONS---
# ad_group ONE-to-MANY
ad_group_db_id = Column(BigInteger, ForeignKey('ad_groups.db_id',
ondelete='CASCADE'), nullable=True)
# Joined Table Inheritance
criterion_sub_type = Column(Unicode(50))
__mapper_args__ = {
'polymorphic_on': criterion_sub_type
}
class AdGroupKeywordModel(AdGroupCriterionModel):
__tablename__ = 'ad_group_keyword'
__mapper_args__ = {'polymorphic_identity': 'Keyword'}
db_id = Column(Integer, ForeignKey('ad_group_criterion.db_id'), primary_key=True)
text = Column(Unicode, nullable=False)
class AdGroupDependenceMixin(object):
_aggad_id = Column(BigInteger, nullable=True)
_agname = Column(Unicode, nullable=True)
#hybrid_property
def ad_group_GAD_id(self):
if self.ad_group is None:
res = self._aggad_id
else:
res = self.ad_group.GAD_id
return res
#ad_group_GAD_id.setter
def ad_group_GAD_id(self, value):
self._aggad_id = value
if value is not None:
self.ad_group = None
#ad_group_GAD_id.expression
def ad_group_GAD_id(cls):
what = case([( cls._aggad_id != None, cls._aggad_id)], else_=AdGroupModel.GAD_id)
return what.label('adgroupgadid_expression')
#hybrid_property
def ad_group_name(self):
if self.ad_group is None:
return self._agname
else:
return self.ad_group.name
#ad_group_name.setter
def ad_group_name(self, value):
self._agname = value
if value is not None:
self.campaign = None
#ad_group_name.expression
def ad_group_name(cls):
what = case([( cls._agname != None, cls._agname)], else_=AdGroupModel.name)
return what.label('adgroupname_expression')
And I load the Keywords objects from the database with the following query:
all_objects1 = self.database.session.query(AdGroupKeywordModel).join(AdGroupModel)\
.options(subqueryload('ad_group'))\
.filter(AdGroupModel.GAD_id!=None)\
.limit(self.options.limit).all()
which returns obejcts of type AdGroupKeywordModel.
Unfortunately every time I try to access the properties of the AdGroupKeywordModel which are in the parent table (AdGroupCriterionModel) a query of this type is emitted:
sqlalchemy.engine.base.Engine
SELECT ad_group_criterion.destination_url AS ad_group_criterion_destination_url, ad_group_criterion.status AS ad_group_criterion_status, ad_group_criterion.ad_group_db_id AS ad_group_criterion_ad_group_db_id, ad_group_criterion.criterion_sub_type AS ad_group_criterion_criterion_sub_type, ad_group_keyword.text AS ad_group_keyword_text
FROM ad_group_criterion JOIN ad_group_keyword ON ad_group_criterion.db_id = ad_group_keyword.db_id
which is strongly compromising the performace.
What I would like to have is that all the attributes for the class AdGroupKeywordModel which are related to the parent (and other classes defined in the relationship) to be loaded with the initial query and be cached for further use. So that when I access them I do not get any overhead from further sqlstatements.
It seems that eager loading is only defined for relationships but not for hierarchies. Is it possible to have this behaviour in sqlalchemy for hierarchies as well?
Thanks
What I see is: only AdGroupModel has a relationship with a lazy= definition (which is the keyword which defines eager loading for relationships), and the query only has a subqueryload('ad_group').
The only point, in which ad_group or AdGroupModel touch with AdGroupKeywordModel is in AdGroupModel.criteria, which has as backref AdGroupCriterionModel.ad_group. I'm not familiar with the subqueryload syntax, but If I would want to eager-load AdGroupCriterionModel.ad_group, I'd define criteria like this:
criteria = relationship(
"AdGroupCriterionModel", backref=backref("ad_group", lazy="subquery"),
lazy="subquery", passive_deletes=True, single_parent=True,
cascade="all, delete, delete-orphan")
The key would be in defining the right lazy also for the backref.

Why 2 queries are executed instead of one?

I have following piece of code:
def detail(request, popular_id):
try:
popular = Popular.objects.get(pk = popular_id)
share = Share.objects.get(isin = popular.isin) #LINE 1
chart_data_json = share.get_chart_data_json()
except Popular.DoesNotExist:
raise Http404
return render(request, 'popular/detail.html', {'popular': popular, 'chart_data': chart_data_json})
In LINE 1 I noticed using debug-toolbar that there are two queries get executed:
SELECT `share_share`.`id`, `share_share`.`symbol`, `share_share`.`isin`, `share_share`.`name`, `share_share`.`market`, `share_share`.`updated` FROM `share_share` WHERE `share_share`.`id` = 1
and
SELECT `share_share`.`id`, `share_share`.`symbol`, `share_share`.`isin`, `share_share`.`name`, `share_share`.`market`, `share_share`.`updated` FROM `share_share` WHERE `share_share`.`isin` = 'US5949181045'
I cannot understand why we need the first query and how to avoid it?
EDIT:
Model definition of share:
class Share(models.Model):
symbol = models.CharField(max_length = 32)
isin = models.CharField(max_length = 12)
name = models.CharField(max_length = 256)
market = models.CharField(max_length = 64)
updated = models.BooleanField(default = False)
def get_chart_data_json(self):
quote_model = create_quote_model(str(self.isin))
data = quote_model.objects.values('date', 'adj_close', 'volume')
chart_data = []
for d in data.iterator():
chart_data.append({'date': d['date'].isoformat(), 'value': d['adj_close'], 'volume': d['volume']})
chart_data_json = json.dumps(chart_data)
return chart_data_json
def __unicode__(self):
return self.isin
Model definition of popular:
class Popular(models.Model):
title = models.CharField(max_length = 120)
text = models.CharField(max_length = 1024)
isin = models.ForeignKey(Share)
def __unicode__(self):
return self.title
First query is evaluated when you access foreign key isin from popular object:
share = Share.objects.get(isin = popular.isin)
Second query gets Share object:
share = Share.objects.get(isin = popular.isin)
If you want just one query at #LINE 1 you should replace it with:
share = popular.isin #LINE 1