I have a sort function on a project I'm working on, where users can create a sort query of all the assets they're working on. When they get the results of their query, I want them to be able to download a .csv of all the objects in the query.
However, when I try to store the query results in a session, I get an error that the results are not JSON serializable. If I don't try to store the query results then the sort runs fine, but then export button won't work since the query results haven't been stored.
In my views:
def sort(request, project_id=1):
thisuser = request.user
project = Project.objects.get(id=project_id)
if Project.objects.filter(Q(created_by=thisuser) | Q(access__give_access_to=thisuser), id=project_id).exists():
permission = 1
else:
permission = None
if Asset.objects.filter(project__id=project_id, unique_id=1):
assets = 1
else:
assets = None
if request.POST:
if request.POST.get('date_start') and request.POST.get('date_end'):
date_start = datetime.strptime(request.POST['date_start'], '%m/%d/%Y')
date_end = datetime.strptime(request.POST['date_end'], '%m/%d/%Y')
q_date = Q(date_produced__range=[date_start, date_end])
else:
q_date = Q(date_produced__isnull=False) | Q(date_produced__isnull=True)
text_fields = {
'asset_type': request.POST.get('asset_type'),
'description': request.POST.get('description'),
'master_status': request.POST.get('master_status'),
'location': request.POST.get('location'),
'file_location': request.POST.get('file_location'),
'footage_format': request.POST.get('footage_format'),
'footage_region': request.POST.get('footage_region'),
'footage_type': request.POST.get('footage_type'),
'footage_fps': request.POST.get('footage_fps'),
'footage_url': request.POST.get('footage_url'),
'stills_credit': request.POST.get('stills_credit'),
'stills_url': request.POST.get('stills_url'),
'music_format': request.POST.get('music_format'),
'music_credit': request.POST.get('music_credit'),
'music_url': request.POST.get('music_url'),
'license_type': request.POST.get('license_type'),
'source': request.POST.get('source'),
'source_contact': request.POST.get('source_contact'),
'source_email': request.POST.get('source_email'),
'source_id': request.POST.get('source_id'),
'source_phone': request.POST.get('source_phone'),
'source_fax': request.POST.get('source_fax'),
'source_address': request.POST.get('source_address'),
'credit_language': request.POST.get('source_language'),
'cost': request.POST.get('cost'),
'cost_model': request.POST.get('cost_model'),
'total_cost': request.POST.get('total_cost'),
'notes': request.POST.get('notes')
}
boolean_fields = {
'used_in_film': request.POST.get('used_in_film'),
'footage_blackandwhite': request.POST.get('footage_blackandwhite'),
'footage_color': request.POST.get('footage_color'),
'footage_sepia': request.POST.get('footage_sepia'),
'stills_blackandwhite': request.POST.get('stills_blackandwhite'),
'stills_color': request.POST.get('stills_color'),
'stills_sepia': request.POST.get('stills_sepia'),
'license_obtained': request.POST.get('license_obtained')
}
q_objects = Q()
for field, value in text_fields.iteritems():
if value:
q_objects = Q(**{field+'__contains': value})
q_boolean = Q()
for field, value in boolean_fields.iteritems():
if value:
q_boolean |= Q(**{field: True})
query_results = Asset.objects.filter(q_date, q_objects, q_boolean)
list(query_results)
request.session['query_results'] = list(query_results)
args = {'query_results': query_results, 'thisuser': thisuser, 'project': project, 'assets': assets}
args.update(csrf(request))
args['query_results'] = query_results
return render_to_response('sort_results.html', args)
else:
args = {'thisuser': thisuser, 'project': project, 'assets': assets}
args.update(csrf(request))
return render_to_response('sort.html', args)
This is the line: "request.session['query_results'] = list(query_results)" that causes it to fail. It also fails if it's "request.session['query_results'] = query_results"
The reason of this error is that you try to assign list on model instances to session. Model instance cannot be serialized to JSON. If you want to pass list of instances of Asset model to session you can do in that way:
query_results = Asset.objects.values('id','name').filter(q_date, q_objects, q_boolean)
You can list necessary model fields in values()
In that case you will have list of dictionaries, not instances. And this list may be assigned to session. But you cannot operate with this dictionaries like instances of class Assign, i.e. you cannot call class methods and so on.
Related
From a Django app, I am able to consume data from a separate Restful API, but what about filtering? Below returns all books and its data. But what if I want to grab only books by an author, date, etc.? I want to pass an author's name parameter, e.g. .../authors-name or /?author=name and return only those in the json response. Is this possible?
views.py
def get_books(request):
response = requests.get('http://localhost:8090/book/list/').json()
return render(request, 'books.html', {'response':response})
So is there a way to filter like a model object?
I can think of three ways of doing this:
Python's filter could be used with a bit of additional code.
QueryableList, which is the closest to an ORM for lists I've seen.
query-filter, which takes a more functional approach.
1. Build-in filter function
You can write a function that returns functions that tell you whether a list element is a match and the pass the generated function into filter.
def filter_pred_factory(**kwargs):
def predicate(item):
for key, value in kwargs.items():
if key not in item or item[key] != value:
return False
return True
return predicate
def get_books(request):
books_data = requests.get('http://localhost:8090/book/list/').json()
pred = filter_pred_factory(**request.GET)
data_filter = filter(pred, books_data)
# data_filter is cast to a list as a precaution
# because it is a filter object,
# which can only be iterated through once before it's exhausted.
filtered_data = list(data_filter)
return render(request, 'books.html', {'books': filtered_data})
2. QueryableList
QueryableList would achieve the same as the above, with some extra features. As well as /books?isbn=1933988673, you could use queries like /books?longDescription__icontains=linux. You can find other functionality here
from QueryableList import QueryableListDicts
def get_books(request):
books_data = requests.get('http://localhost:8090/book/list/').json()
queryable_books = QueryableListDicts(books_data)
filtered_data = queryable_books.filter(**request.GET)
return render(request, 'books.html', {'books':filtered_data})
3. query-filter
query-filter has similar features but doesn't copy the object-orient approach of an ORM.
from query_filter import q_filter, q_items
def get_books(request):
books_data = requests.get('http://localhost:8090/book/list/').json()
data_filter = q_filter(books_data, q_items(**request.GET))
# filtered_data is cast to a list as a precaution
# because q_filter returns a filter object,
# which can only be iterated through once before it's exhausted.
filtered_data = list(data_filter)
return render(request, 'books.html', {'books': filtered_data})
It's worth mentioning that I wrote query-filter.
I have SQLAlchemy models on top of the MySQL db. I need to query almost all models (string or text fields) and find everything that contains a specific substring. And also, apply common filtering like object_type=type1. For exsmple:
class Model1(Model):
name = Column(String(100), nullable=False, unique=True)
version = Column(String(100))
description = Column(String(100))
updated_at = Column(TIMESTAMP(timezone=True))
# other fields
class Model2(Model):
name = Column(String(100), nullable=False, unique=True)
version = Column(String(100))
description = Column(String(100))
updated_at = Column(TIMESTAMP(timezone=True))
# other fields
class Model3(Model):
name = Column(String(100), nullable=False, unique=True)
version = Column(String(100))
description = Column(String(100))
updated_at = Column(TIMESTAMP(timezone=True))
# other fields
And then do query something like:
db.query(
Model1.any_of_all_columns.contains('sub_string') or
Model2.any_of_all_columns.contains('sub_string') or
Model3.any_of_all_columns.contains('sub_string')
).all()
Is it possible to build such an ORM query in one SQL to the db and dynamically add Model(table) names and columns?
For applying common filtering for all the columns, you can subscribe to sqlachemy events as following:
#event.listens_for(Query, "before_compile", retval=True)
def before_compile(query):
for ent in query.column_descriptions:
entity = ent['entity']
if entity is None:
continue
inspect_entity_for_mapper = inspect(ent['entity'])
mapper = getattr(inspect_entity_for_mapper, 'mapper', None)
if mapper and has_tenant_id:
query = query.enable_assertions(False).filter(
ent['entity’].object == object)
return query
This function will be called whenever you do Model.query() and add filter for your object.
I eventually gave up and did one big loop in which I make a separate request for each model:
from sqlalchemy import or_
def db_search(self, model, q, object_ids=None, status=None, m2m_ids=None):
"""
Build a query to the db for given model using 'q' search substring
and filter it by object ids, its status and m2m related model.
:param model: a model object which columns will be used for search.
:param q: the query substring we are trying to find in all
string/text columns of the model.
:param object_ids: list of ids we want to include in the search.
If the list is empty, the search query will return 0 results.
If object_ids is None, we will ignore this filter.
:param status: name of object status.
:param m2m_ids: list of many-to-many related object ids.
:return: sqlalchemy query result.
"""
# Filter out private columns and not string/text columns
string_text_columns = [
column.name for column in model.__table__.columns if
isinstance(column.type, (db.String, db.Text))
and column.name not in PRIVATE_COLUMN_NAMES
]
# Find only enum ForeignKey columns
foreign_key_columns = [
column.name for column in model.__table__.columns if
column.name.endswith("_id") and column.name in ENUM_OBJECTS
)
]
query_result = model.query
# Search in all string/text columns for the required query
# as % LIKE %
if q:
query_result = query_result.join(
# Join related enum tables for being able to search in
*[enum_tables_to_model_map[col]["model_name"] for col in
foreign_key_columns]
).filter(
or_(
# Search 'q' substring in all string/text columns
*[
getattr(model, col_name).like(f"%{q}%")
for col_name in string_text_columns
],
# Search 'q' substring in the enum tables
*[
enum_tables_to_model_map[col]["model_field"]
.like(f"%{q}%") for col in foreign_key_columns
]
)
)
# Apply filter by object ids if given and it's not None.
# If the object ids filter exist but it's empty, we should
# return an empty result
if object_ids is not None:
query_result = query_result.filter(model.id.in_(object_ids))
# Apply filter by status if given and if the model has the status
# column
if status and 'status_id' in model.__table__.columns:
query_result = query_result.filter(model.status_id == status.id)
if m2m_ids:
query_result = query_result.filter(
model.labels.any(Label.id.in_(m2m_ids)))
return query_result.all()
And call it:
result = {}
for model in db.Model._decl_class_registry.values():
# Search only in the public tables
# sqlalchemy.ext.declarative.clsregistry._ModuleMarker object
# located in the _decl_class_registry that is why we check
# instance type and whether it is subclass of the db.Model
if isinstance(model, type) and issubclass(model, db.Model) \
and model.__name__ in PUBLIC_MODEL_NAMES:
query_result = self.db_search(
model, q, object_ids.get(model.__name__), status=status,
m2m_ids=m2m_ids)
result[model.__tablename__] = query_result
This is far from the best solution, but it works for me.
I have this view
def view_involved_people(request):
schedule = request.POST['schedule']
query = Schedule.objects.get(pk=schedule)
serialized = serializers.serialize('json', query)
data = {'people': serialized}
return JsonResponse(data)
It displays that the object is not iterable. I think it is because I am only getting one instance of the object. However, how can I prevent this error and get this data from the view?
I have tried using .filter() but when I call data.attribute_name, it does not display the value
You have to use filter in you case:
def view_involved_people(request):
schedule = request.POST['schedule']
query = Schedule.objects.filter(pk=schedule)
serialized = serializers.serialize('json', query)
data = {'people': serialized}
return JsonResponse(data)
This code seems to work fine when I used Django console and just print it.
reference = FloodHazard.objects.filter(hazard='High')
ids = reference.values_list('id', flat=True)
for myid in ids:
getgeom = FloodHazard.objects.get(id=myid).geom
response = BuildingStructure.objects.filter(geom__intersects=getgeom).values(
'brgy_locat').annotate(counthigh=Count('brgy_locat'))
print response
I was able to show all the values, but when using HttpResponse, it returns an empty set. What is the proper way of returning JSON data from a queryset? So far, tried this:
reference = FloodHazard.objects.filter(hazard='High')
ids = reference.values_list('id', flat=True)
response = {}
for myid in ids:
getgeom = FloodHazard.objects.get(id=myid).geom
response['high'] = BuildingStructure.objects.filter(geom__intersects=getgeom).values(
'brgy_locat').annotate(counthigh=Count('brgy_locat'))
json_post = ujson.dumps(list(response))
return HttpResponse(json_post, content_type='application/json')
There is no much sense in your code. You assign all querysets to the single key in the response dict. You should use a list for this purpose:
As far as I understand the code should be something like this:
response = []
for myid in ids:
getgeom = FloodHazard.objects.get(id=myid).geom
response.extend(BuildingStructure.objects.filter(geom__intersects=getgeom)
.values('brgy_locat')
.annotate(counthigh=Count('brgy_locat')))
json_post = ujson.dumps(response)
If you want to return a hazard level as well as the list of buildings then you can return a dict:
json_post = ujson.dumps({'hazard': 'high', 'buildings': response})
I have a ndb.Model that I want to convert to JSON.
class Users(ndb.Model):
username = ndb.StringProperty(indexed=True)
password= ndb.StringProperty(indexed=True)
created_at = ndb.DateTimeProperty(auto_now_add=True)
user = Users.query(Users.username==username).get()
rv = json.dumps(user.to_dict())
print(rv)
It throws this error:
TypeError: datetime.datetime(2013, 11, 24, 3, 40, 15) is not JSON serializable
Most of the solutions here are for db.Model and are fairly outdated.
sdk version 1.9.10
You can extend Property classes to handle special cases. It is applicable for any property type.
from google.appengine.ext import ndb
class DateTimeProperty(ndb.DateTimeProperty):
# Override to allow JSON serialization
def _get_for_dict(self, entity):
value = super(DateTimeProperty, self)._get_for_dict(entity);
return value.isoformat()
Then use it in your model:
class Users(ndb.Model):
username = ndb.StringProperty(indexed=True)
password= ndb.StringProperty(indexed=True)
created_at = DateTimeProperty(auto_now_add=True)
And to_dict() as you would normally do:
user = Users.query(Users.username==username).get()
user.to_dict()
You would need a custom "to JSON" converter that handles formats not natively supported by JSON.
I am using something like the following code that handles most situations for me.
def to_json(self, o):
if isinstance(o, list):
return [self.to_json(l) for l in o]
if isinstance(o, dict):
x = {}
for l in o:
x[l] = self.to_json(o[l])
return x
if isinstance(o, datetime.datetime):
return o.isoformat()
if isinstance(o, ndb.GeoPt):
return {'lat': o.lat, 'lon': o.lon}
if isinstance(o, ndb.Key):
return o.urlsafe()
if isinstance(o, ndb.Model):
dct = o.to_dict()
dct['id'] = o.key.id()
return self.to_json(dct)
return o
So in my case I am also taking care of some other things like GeoPt, and adding an ID field to all ndb.Models but for your case all you'd need would be:
if isinstance(o, datetime.datetime):
return o.isoformat()
but I am guessing (not really sure) you would then get a key error as well so you'd also need
if isinstance(o, ndb.Key):
return o.urlsafe()
In case if you didn't need the created_at field, you could simply exclude it like
rv = json.dumps(user.to_dict(exclude=['created_at']))
Another possible solution - inspired from this answer - is to override to_dict() in your User class:
class User(ndb.Model):
username = ndb.StringProperty(indexed=True)
password = ndb.StringProperty(indexed=False)
created_at = DateTimeProperty(auto_now_add=True)
def to_dict(self):
result = super(User, self).to_dict()
result['created_at'] = self.created_at.isoformat()
return result