#necessary import goes here
engine = sqlalchemy.create_engine('mysql://root#127.0.0.1/test',echo=False)
print 'Engine created'
connection=engine.connect()
metadata=MetaData(engine)
metadata.bind=engine
Session = sessionmaker(bind=engine)
session = Session()
mapping = Table('mapping',metadata,autoload=True)
class Mapping(object):
pass
MappingMapper=mapper(Mapping,mapping)
Now i am able to write basic query for insert,update,delete,filter etc.
Q:1 I need to write complex query, where i do derive new columns based on existing columns. Ex. ColA,ColB is there on table, ColC is not part of table structure.
Select (ColA+ColB) as ColC from table where ColC > 50 order by ColC.
I am clueless how to convert above like query with SqlAlchemy. How to map, how to retrieve.
The easiest is to useHybrid Attributes.
In your case, just change the declaration of the class to the following:
from sqlalchemy.ext.hybrid import hybrid_property
class Mapping(object):
#hybrid_property
def ColC(self):
return self.ColA + self.ColB
Then the query:
qry = session.query(Mapping).filter(Mapping.ColC > 80)
will generate SQL:
SELECT mapping.id AS mapping_id, ...
FROM mapping
WHERE mapping."ColA" + mapping."ColB" > ?
Related
I have three tables
User
Device
Log
I want to filter the logs based on devices and logs. I'm using the following querying which iterates over the users and devices in order to get the logs. I feel this will become a performance hit. How to reduce the number of database hits?
for user_obj in User.objects.all():
device_qs = Device.objects.filter(user=user_obj)
if device_qs.exists():
for device_obj in device_qs:
log_count = Log.objects.filter(user=user_obj, device=device_obj, created_at__range(from_date, to_date)).count()
If you only need the log count per user and device (which is what you get from the code you posted), you can get that in just one query:
from django.db.models import Count
logs = (Log.objects
.filter(created_at__range = (from_date, to_date))
.values('user', 'device')
.annotate(log_count=Count('device'))
)
You can modify the query to include any attributes of the user and device models that you need:
.values('user__last_name', 'device__name') # etc.
You can also order the dataset by appending order_by() at the end to be able to iterate over it in the desired order:
.order_by('user__last_name', '-log_count')
What I would do is create a "proxy model" that references a view in your MySQL instance
The view would look like this:
SELECT
t1.*,
t2.*,
t3.*
FROM users t1
RIGHT JOIN device t2 (ON t1.id=t2.user_id)
RIGHT JOIN log t3 (ON t3.device_id=t2.id);
Now to create a proxy model, do this:
class SomeModel(models.Model):
# all fields from the 3 tables here
class Meta:
db_table = 'yourViewNameHere'
managed = False # this keeps django from creating the table
then python manage.py makemigrations + python manage.py migrate as usual
Now, to access the the data you need, you would do something like this:
from django.db import connection
sql = "SELECT * FROM your_view WHERE some_date_column > 'foo' AND some_date_column < 'bar' "
with connection.cursor() as cur:
cur.execute(sql)
data = cur.fetchall()
print(data)
Note that if you are passing parameters to the raw sql query, you should always pass them like this to avoid sql injection:
sql = "SELECT * FROM your_view WHERE some_date_column > %s AND some_date_column < %s"
params = ('foo', 'bar')
with connection.cursor() as cur:
cur.execute(sql, params)
data = cur.fetchall()
I have two django-models
class ModelA(models.Model):
title = models.CharField(..., db_column='title')
text_a = models.CharField(..., db_column='text_a')
other_column = models.CharField(/*...*/ db_column='other_column_a')
class ModelB(models.Model):
title = models.CharField(..., db_column='title')
text_a = models.CharField(..., db_column='text_b')
other_column = None
Then I want to merge the two querysets of this models using union
ModelA.objects.all().union(ModelB.objects.all())
But in query I see
(SELECT
`model_a`.`title`,
`model_a`.`text_a`,
`model_a`.`other_column`
FROM `model_a`)
UNION
(SELECT
`model_b`.`title`,
`model_b`.`text_b`
FROM `model_b`)
Of course I got the exception The used SELECT statements have a different number of columns.
How to create the aliases and fake columns to use union-query?
You can annotate your last column to make up for column number mismatch.
a = ModelA.objects.values_list('text_a', 'title', 'other_column')
b = ModelB.objects.values_list('text_a', 'title')
.annotate(other_column=Value("Placeholder", CharField()))
# for a list of tuples
a.union(b)
# or if you want list of dict
# (this has to be the values of the base query, in this case a)
a.union(b).values('text_a', 'title', 'other_column')
In SQL query, we can use NULL to define the remaining columns/aliases
(SELECT
`model_a`.`title`,
`model_a`.`text_a`,
`model_a`.`other_column`
FROM `model_a`)
UNION
(SELECT
`model_b`.`title`,
`model_b`.`text_b`,
NULL
FROM `model_b`)
In Django, union operations needs to have same columns, so with values_list you can use those specific columns only like this:
qsa = ModelA.objects.all().values('text_a', 'title')
qsb = ModelB.objects.all().values('text_a', 'title')
qsa.union(qsb)
But there is no way(that I know of) to mimic NULL in union in Django. So there are two ways you can proceed here.
First One, add an extra field in your Model with name other_column. You can put the values empty like this:
other_column = models.CharField(max_length=255, null=True, default=None)
and use the Django queryset union operations as described in here.
Last One, the approach is bit pythonic. Try like this:
a = ModelA.objects.values_list('text_a', 'title', 'other_column')
b = ModelB.objects.values_list('text_a', 'title')
union_list = list()
for i in range(0, len(a)):
if b[i] not in a[i]:
union_list.append(b[i])
union_list.append(a[i])
Hope it helps!!
I'm in the middle of converting an old legacy PHP system to Flask + SQLAlchemy and was wondering how I would construct the following:
I have a model:
class Invoice(db.Model):
paidtodate = db.Column(DECIMAL(10,2))
fullinvoiceamount = db.Column(DECIMAL(10,2))
invoiceamount = db.Column(DECIMAL(10,2))
invoicetype = db.Column(db.String(10))
acis_cost = db.Column(DECIMAL(10,2))
The query I need to run is:
SELECT COUNT(*) AS the_count, sum(if(paidtodate>0,paidtodate,if(invoicetype='CPCN' or invoicetype='CPON' or invoicetype='CBCN' or invoicetype='CBON' or invoicetype='CPUB' or invoicetype='CPGU' or invoicetype='CPSO',invoiceamount,
fullinvoiceamount))) AS amount,
SUM(acis_cost) AS cost, (SUM(if(paidtodate>0,paidtodate,invoiceamount))-SUM(acis_cost)) AS profit FROM tblclientinvoices
Is there an SQLAlchemyish way to construct this query? - I've tried googling for Mysql IF statments with SQlAlchemy but drew blanks.
Many thanks!
Use func(documentation) to generate SQL function expression:
qry = select([
func.count().label("the_count"),
func.sum(func.IF(
Invoice.paidtodate>0,
Invoice.paidtodate,
# #note: I prefer using IN instead of multiple OR statements
func.IF(Invoice.invoicetype.in_(
("CPCN", "CPON", "CBCN", "CBON", "CPUB", "CPGU", "CPSO",)
),
Invoice.invoiceamount,
Invoice.fullinvoiceamount)
)
).label("amount"),
func.sum(Invoice.acis_cost).label("Cost"),
(func.sum(func.IF(
Invoice.paidtodate>0,
Invoice.paidtodate,
Invoice.invoiceamount
))
- func.sum(Invoice.acis_cost)
).label("Profit"),
],
)
rows = session.query(qry).all()
for row in rows:
print row
I have a few tables shown below that I would like to join on columns that are not foreign keys to each other's tables and then have access to the columns of both. Here are the classes:
class Yi(db.Model):
year = db.Column(db.Integer(4), primary_key=True)
industry_id = db.Column(db.String(5), primary_key=True)
wage = db.Column(db.Float())
complexity = db.Column(db.Float())
class Ygi(db.Model, AutoSerialize):
year = db.Column(db.Integer(4), primary_key=True)
geo_id = db.Column(db.String(8), primary_key=True)
industry_id = db.Column(db.String(5), primary_key=True)
wage = db.Column(db.Float())
So, what I would like to get are the columns of both tables joined by the IDs I specify, in this case Year and industry_id. Is this possible? Here is the SQL I've written to achieve this...
SELECT
yi.complexity, ygi.*
FROM
yi, ygi
WHERE
yi.year = ygi.year and
yi.industry_id = ygi.industry_id
One dirty way is :
q=session.query(Ygi,Yi.complexity).\
filter(Yi.year==Ygi.year).\
filter(Yi.industry_id==Ygi.industry_id)
Which gives you :
SELECT ygi.year AS ygi_year, ygi.geo_id AS ygi_geo_id,
ygi.industry_id AS ygi_industry_id, ygi.wage AS ygi_wage,
yi.complexity AS yi_complexity
FROM ygi, yi
WHERE yi.year = ygi.year
AND yi.industry_id = ygi.industry_id
I find this dirty because it does not use the join() method.
You can figure out how to use the join() with the SQLAlchemy documentation
Then, you can choose to use a virtual model : see answer of TokenMacGuy in this question Mapping a 'fake' object in SQLAlchemy.
It will be a good solution.
Or you will just have a YiYgi class that will not be a sqlalchemy.Base derived class but just an object. It more a "hand-fashion" way to do it.
The class will have a classmethod get() method that will:
call the query you build just before,
call the init with the returned request lines and build up one instance per line
This is an example :
class YiYgi(object):
def __init__(self,year, geo_id, industry_id, wage, complexity):
# Initialize all your fields
self.year = year
self.geo_id = geo_id
self.industry_id = industry_id
self.wage = wage + 100 # You can even make some modifications to the values here
self.complexity = complexity
#classmethod
def get_by_year_and_industry(cls, year, industry_id):
""" Return a list of YiYgi instances, void list if nothing available """
q = session.query(Ygi,Yi.complexity).\
filter(Yi.year==Ygi.year).\
filter(Yi.industry_id==Ygi.industry_id)
results = q.all()
yiygi_list = []
for result in results:
# result is a tuple with (YGi instance, Yi.complexity value)
ygi_result = result[0]
yiygi = YiYgi(ygi_result.ygi_year,
ygi_result.geo_id,
ygi_result.industry_id,
ygi_result.wage,
result[1])
yiygi_list.append(yiygi)
return yiygi_list
I try to do the follwoing request in SQLAlchemy (ORM) :
SELECT id, ref_prog FROM stepvand_1c_1t.equipment_day_hour
WHERE id IN (SELECT id FROM stepvand_1c_1t.equipment WHERE equipment_type='L')
I did :
subq = session.query(Equipment)
subq = subq.filter(Equipment.equipment_type == "L").subquery()
query = session.query(EquipmentDayHour)
query = query.filter(EquipmentDayHour.id.in_(subq))
But that doesn't work...
Python tells me that the subrequest has too many columns.
I think you should only change one line of your sample code:
# error: includes all columns of Equipment
`subq = session.query(Equipment)`
# correct: include only ID column
`subq = session.query(Equipment.id)`
However, I believe that you can do this without subquery:
query = (session.query(EquipmentDayHour).
# version-1: if you have a relationship between EquipmentDayHour and Equipment
join(Equipment).
# version-2: if you do not have such relationship
#join(Equipment, EquipmentDayHour.id==Equipment.id).
filter(Equipment.equipment_type == "L")
)