I want to load below json data in my Model.
{
"99popularity": 79.0,
"director": "William Cottrell",
"genre": [
"Animation",
" Family",
" Fantasy",
" Musical",
" Romance"
],
"imdb_score": 7.9,
"name": "Snow White and the Seven Dwarfs"
},
{
"99popularity": 84.0,
"director": "Stanley Kubrick",
"genre": [
"Adventure",
" Mystery",
" Sci-Fi"
],
"imdb_score": 8.4,
"name": "2001 : A Space Odyssey"
},
I have create two models using reference to json data
class Genre(models.Model):
name = models.CharField(max_length=30)
class Movie(models.Model):
popularity = models.FloatField(max_length=10)
director = models.CharField(max_length=30)
genre = models.ManyToManyField(Genre)
imdb_score = models.FloatField(max_length=10)
name = models.CharField(max_length=30)
But in Genre Model i don't have any data and in json in genre section their is no id instead name. How can i load that data in my model. Please Help.
You can use the get_or_create method, but you have to make the name field unique. To create two models using reference to json data, I would use a custom class method like this:
class Genre(models.Model):
name = models.CharField(max_length=30, unique=True) # make unique
class Movie(models.Model):
popularity = models.FloatField(max_length=10)
director = models.CharField(max_length=30)
genre = models.ManyToManyField(Genre)
imdb_score = models.FloatField(max_length=10)
name = models.CharField(max_length=30)
#classmethod
def create(cls, **kwargs):
movie = cls.objects.create(
popularity=kwargs['99popularity'],
director=kwargs['director'],
imdb_score=kwargs['imdb_score'],
name=kwargs['name']
)
for genre_name in kwargs['genre']:
genre, created = Genre.objects.get_or_create(name=genre_name)
movie.genre.add(genre)
return movie
Assuming you converted your json data to a string you can do this:
import json
from .models import Movie
# you can also keep this inside a view
with open('movie_data.json', encoding='utf-8') as data_file:
json_data = json.loads(data_file.read())
for movie_data in json_data:
movie = Movie.create(**movie_data)
# movie and genres created
Related
Given the following codes:
from flask import Flask, jsonify
from flask_sqlalchemy import SQLAlchemy
app = Flask(__name__)
app.config["SQLALCHEMY_ECHO"] = True
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///test.db"
db = SQLAlchemy(app)
class File(db.Model):
__tablename__ = "file"
_id = db.Column(db.String, primary_key=True)
file_name = db.Column(db.String)
category_id = db.Column(db.String, db.ForeignKey("category._id"))
def as_dict(self):
return {c.name: getattr(self, c.name) for c in self.__table__.columns}
class Category(db.Model):
__tablename__ = "category"
_id = db.Column(db.String, primary_key=True)
name = db.Column(db.String)
files = db.relationship("File", backref="category")
def as_dict(self):
return {c.name: getattr(self, c.name) for c in self.__table__.columns}
db.drop_all()
db.create_all()
categories0 = Category(_id="aca50a26-5d3f-4c4d-872b-83b663d5304f",name="Apple")
files0 = File(_id="8a95ba11-e2aa-407d-bac9-609e6c559731",file_name="8a95ba11-e2aa-407d-bac9-609e6c559731.jpg",category=categories0)
db.session.add_all([categories0,files0])
db.session.commit()
results=db.session.query(File).join(Category, File.category).filter(Category._id=="aca50a26-5d3f-4c4d-872b-83b663d5304f").all()
#app.route('/print')
def printMsg():
return jsonify([c.as_dict() for c in results])
if __name__ == '__main__':
app.run(debug=True)
When I call the endpoint /print, it returns
[
{
"_id": "8a95ba11-e2aa-407d-bac9-609e6c559731",
"category_id": "aca50a26-5d3f-4c4d-872b-83b663d5304f",
"file_name": "8a95ba11-e2aa-407d-bac9-609e6c559731.jpg"
}
]
But I need the category_name as output
[
{
"_id": "8a95ba11-e2aa-407d-bac9-609e6c559731",
"category_id": "aca50a26-5d3f-4c4d-872b-83b663d5304f",
"category_name": "Apple",
"file_name": "8a95ba11-e2aa-407d-bac9-609e6c559731.jpg"
}
]
How should I achieve that?
Here's code working as you requested. There is some other help I'll offer in here...
You code iterated columns. That's the Database's view of things. You should iterate the attributes of the model. In many cases, they're the same for fields (not always). But for relationships, you HAVE to be looking at the model.
Instead of pulling the related attribute into the dictionary of fields, I offer a 2nd solution to NEST the related fields in the response. As your models grow and scale, you will appreciate this more than you may now.
I changed some formatting for PEP8. Even for SO posts, it helps readability and its always good practice.
I would STRONGLY consider you DO NOT use either of these solutions. Serialization is a tricky process. For simple fields/relationships like your example, it may not seem that way. Again, as you scale, you will be handling a lot of edge cases and writing code that's been written and tested many time before. Consider using Marshmallow / Flask-Marshmallow. It's a great library that makes serialization and relationship nesting trivial.
from flask import Flask, jsonify
from flask_sqlalchemy import SQLAlchemy
import sqlalchemy as sa
app = Flask(__name__)
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:'
db = SQLAlchemy(app)
class File(db.Model):
__tablename__ = 'file'
_id = db.Column(
db.String,
primary_key=True,
)
file_name = db.Column(db.String, )
category_id = db.Column(
db.String,
db.ForeignKey('category._id'),
)
def as_dict(self):
"""Return serialzed attributes + related category name"""
serialized = {
attr.key: getattr(self, attr.key)
for attr in sa.orm.class_mapper(self.__class__).iterate_properties
if isinstance(attr, sa.orm.properties.ColumnProperty)
}
serialized['category_name'] = self.category.name
return serialized
def as_dict_with_relationships(self):
"""Return serialzed attributes + nested relationships"""
serialized = {}
for attr in sa.orm.class_mapper(self.__class__).iterate_properties:
if isinstance(attr, sa.orm.properties.ColumnProperty):
serialized[attr.key] = getattr(self, attr.key)
elif isinstance(attr, sa.orm.relationships.RelationshipProperty):
serialized[attr.key] = getattr(self, attr.key).as_dict()
else:
print(f'not handling {attr}, {type(attr)}')
return serialized
class Category(db.Model):
__tablename__ = 'category'
_id = db.Column(
db.String,
primary_key=True,
)
name = db.Column(db.String, )
files = db.relationship(
'File',
backref='category',
)
def as_dict(self):
return {
c.name: getattr(self, c.name)
for c in self.__table__.columns
}
db.drop_all()
db.create_all()
categories0 = Category(
_id='aca50a26-5d3f-4c4d-872b-83b663d5304f',
name='Apple',
)
files0 = File(
_id='8a95ba11-e2aa-407d-bac9-609e6c559731',
file_name='8a95ba11-e2aa-407d-bac9-609e6c559731.jpg',
category=categories0,
)
db.session.add_all([categories0, files0])
db.session.commit()
#app.route('/print')
def print_msg():
"""Return serialized results- top level attributes + explicit other data"""
results = db.session.query(File) \
.join(Category, File.category) \
.filter(
Category._id == 'aca50a26-5d3f-4c4d-872b-83b663d5304f',
) \
.all()
return jsonify([
record.as_dict()
for record in results
])
def print_msg2():
"""Return serialized results- top level attributes + nested relationships"""
results = db.session.query(File) \
.join(Category, File.category) \
.filter(
Category._id == 'aca50a26-5d3f-4c4d-872b-83b663d5304f',
) \
.all()
return jsonify([
record.as_dict_with_relationships()
for record in results
])
if __name__ == '__main__':
app.run(debug=True)
How do I send a Django model and it's related (_set)s via Ajax?
class Zoo(models.Model):
id = models.AutoField(primary_key=True)
details = models.TextField(blank=True) # HUGE AMOUNT OF DATA
animals_json = models.TextField(blank=True) # Information on what animals are in Zoo
class Animals(models.Model):
id = models.AutoField(primary_key=True)
zoo = models.ForeignKey(Zoo, on_delete=models.PROTECT)
animal_name = models.TextField(blank=True)
zoo = Zoo.objects.filter(pk=5)
animals = []
for animal in json.loads(zoo.animals_json):
animals.append(Animals(zoo=zoo, animal_name=animal['name']))
# Now I have a Zoo model and a list of animals.
# My Zoo model has a animal_set (RelatedManager) object
Now, I would like to send a JSON encoded Zoo via Ajax to front end. How do I do this? Option 1)
zoo_json = serializers.serialize('json', [zoo])
return HttpResposne(zoo_json)
This won't work because then I'm not sending any information about the animals. (Yes I realized it's encoded in the animals_json, but I want to just send clean objects via AJAX and not parse it with a front end loop). Something like this seems very messy as well because now the objects are seperate from one another:
zoo_json = serializers.serialize('json', [zoo])
animals_json = serializers.serialize('json', animals)
data_to_send_via_ajaz = {'zoo' : zoo_json, 'animals' : animals_json}
return HttpResposne(zoo_json)
My ideal object looks like this:
zoo_model_in_json_form [
animal_model_in_json_form,
animal_model_in_json_form,
animal_model_in_json_form,
...
]
return HttpResposne(zoo_model_in_json_form)
This way I can loop through all the animals very easily. Any ideas?
You can always use your own method of serializing the objects using dict objects and finally using json.dumps to dump the dict object as a string with response content_type set to application/json.
Solution:
import json
zoo = Zoo.objects.filter(pk=5).first() # adding `first` call to get a single zoo
serialized_zoo_data = dict()
# serialize zoo with its animals nested in it
# Serialized zoo with animals:
# {
# "id": 1,
# "details": "blah blah",
# "animals": [{
# "id": 222,
# "name": "Gorilla"
# }, {
# "id": 11,
# "name": "Camel"
# }]
# }
if zoo is not None:
# get all animals belonging to a `Zoo` object
zoo_animals = zoo.animal_set.all()
serialized_zoo_data["id"] = zoo.id
serialized_zoo_data["details"] = zoo.details
serialized_zoo_data["animals"] = []
for animal in zoo_animals:
animal = dict()
animal["id"] = animal.id
animal["name"] = animal.animal_name
serialized_zoo_data["animals"].append(animal)
return HttpResponse(json.dumps(serialized_zoo_data), content_type="application/json")
I currently have two classes in Python like these ones
class person:
age=""
name=""
ranking = {}
def addRanking():
#Do Whatever treatment and add to the ranking dict
class ranking:
semester = ""
position = ""
gpa = ""
I have my list of person as a dictionary called dictP json.dumps() this dictionary but it seems that it doesn't work. Here is my function to dump to JSON
def toJson():
jsonfile = open('dict.json', 'w')
print(json.dump(listP, jsonfile))
I get the famous: is not JSON serializable.
Would you know what I can do to help this problem. I thought that having two dictionaries (which are serializable) would avoid this kind of issue, but apparently not.
Thanks in advance
Edit:
Here is an example (typed on my phone sorry for typos, I'm not sure it does run but it's so you get the idea):
class person:
age=""
name=""
ranking = {}
def __init__(self, age, name):
self.age = age
self.name = name
self.ranking = {}
def addRanking(self,semester,position,gpa):
#if the semester is not already present in the data for that person
self.ranking[semester] = make_ranking(semester,position,gpa)
class ranking:
semester = ""
position = ""
gpa = ""
def __init__(self, semester, position, gpa):
self.semester = semester
self.position = position
self.gpa = gpa
dictP = {}
def make_person(age, name):
# Some stuff happens there
return person(age,name)
def make_ranking(semester,postion,gpa):
#some computation there
return ranking(semester,position,gpa)
def pretending_to_read_csv():
age = 12
name = "Alice"
p = make_person(age, name)
dictP["1"] = p
age = 13
name = "Alice"
p = make_person(age, name)
dictP["2"] = p
#We read a csv for ranking that gives us an ID
semester = 1
position = 4
gpa = 3.2
id = 1
dictP["1"].addRanking(semester, position, gpa)
semester = 2
position = 4
gpa = 3.2
id = 1
dictP["1"].addRanking(semester, position, gpa)
For a dictionary to be serializable, note that all the keys & values in that dictionary must be serializable as well. You did not show us what listP contains, but I'm guessing it's something like this:
>>> listP
[<__main__.person instance at 0x107b65290>, <__main__.person instance at 0x107b65368>]
Python instances are not serializable.
I think you want a list of dictionaries, which would look like this:
>>> listP
[{'ranking': {}, 'age': 10, 'name': 'fred'}, {'ranking': {}, 'age': 20, 'name': 'mary'}]
This would serialize as you expect:
>>> import json
>>> json.dumps(listP)
'[{"ranking": {}, "age": 10, "name": "fred"}, {"ranking": {}, "age": 20, "name": "mary"}]'
UPDATE
(Thanks for adding example code.)
>>> pretending_to_read_csv()
>>> dictP
{'1': <__main__.person instance at 0x107b65368>, '2': <__main__.person instance at 0x107b863b0>}
Recall that user-defined classes cannot be serialized automatically. It's possible to extend the JSONEncoder directly to handle these cases, but all you really need is a function that can turn your object into a dictionary comprised entirely of primitives.
def convert_ranking(ranking):
return {
"semester": ranking.semester,
"position": ranking.position,
"gpa": ranking.gpa}
def convert_person(person):
return {
"age": person.age,
"name": person.name,
"ranking": {semester: convert_ranking(ranking) for semester, ranking in person.ranking.iteritems()}}
One more dictionary comprehension to actually do the conversion and you're all set:
>>> new_dict = {person_id: convert_person(person) for person_id, person in dictP.iteritems()}
>>> from pprint import pprint
>>> pprint(new_dict)
{'1': {'age': 12,
'name': 'Alice',
'ranking': {1: {'gpa': 3.2, 'position': 4, 'semester': 1},
2: {'gpa': 3.2, 'position': 4, 'semester': 2}}},
'2': {'age': 13, 'name': 'Alice', 'ranking': {}}}
Since no user-defined objects are stuffed in there, this will serialize as you hope:
>>> json.dumps(new_dict)
'{"1": {"ranking": {"1": {"position": 4, "semester": 1, "gpa": 3.2}, "2": {"position": 4, "semester": 2, "gpa": 3.2}}, "age": 12, "name": "Alice"}, "2": {"ranking": {}, "age": 13, "name": "Alice"}}'
You can try calling json.dump on the .__dict__ member of your instance. You say that you have a list of person instances so try doing something like this:
listJSON = []
for p in listP
#append the value of the dictionary containing data about your person instance to a list
listJSON.append(p.__dict__)
json.dump(listJSON, jsonfile)
If you are storing your person instances in a dictionary like so: dictP = {'person1': p1, 'person2': p2} this solution will loop through the keys and change their corresponding values to the __dict__ member of the instance:
for key in dictP:
dictP[key] = dictP[key].__dict__
json.dump(dictP, jsonfile)
I'm starting with instances of Order object and trying to transform them into a JSON format, which will be used to update a table. I'm new to Django-specific code constructs.
My chosen approach is:
Load the data from database as multiple Order instances
Transform the Order instances into an intermediary Table object (the format is given)
Serialize the Table object into JSON
I have gone quite far so far, but I can't make the whole application run, except for running it in the Python shell. I have the models and serializers already in place.
Can someone offer his help how to mark all Order instances at once and transform them to Table in one API call and anything else that is missing in this simple example?
models.py:
# Order corresponds to Line in the Table
class Order(models.Model):
doc = models.CharField(max_length=200, blank=True, null=True)
order = models.CharField(max_length=200, blank=True, null=True)
nothing = models.CharField(max_length=200, blank=True, null=True)
def __str__(self):
return self.order
class Table(models.Model):
pass
class Column(models.Model):
data = models.CharField(max_length=200, blank=True, null=True)
table = models.ForeignKey(Table)
def __str__(self):
return self.data
class Line(models.Model):
doc = models.CharField(max_length=200, blank=True, null=True)
order = models.CharField(max_length=200, blank=True, null=True)
nothing = models.CharField(max_length=200, blank=True, null=True)
table = models.ForeignKey(Table)
def __str__(self):
return self.order
serializers.py:
class TableSerializer(serializers.ModelSerializer):
columns = ColumnSerializer(many=True)
lines = LineSerializer(many=True)
class Meta:
model = Table
fields = [
'columns',
'lines'
]
class ColumnSerializer(serializers.ModelSerializer):
class Meta:
model = Column
fields = [
'data'
]
class LineSerializer(serializers.ModelSerializer):
class Meta:
model = Line
fields = [
'doc',
'order',
'nothing'
]
For this relational data:
doc order nothing
564251422 564210 5648
546546545 98745 4668
JSON output should be:
{
"columns": [
{
"data": "doc"
},
{
"data": "order"
},
{
"data": "nothing"
}
],
"lines": [
{
"doc": "564251422",
"nothing": 0.0,
"order": "56421"
},
{
"doc": "546546545",
"nothing": 0.0,
"order": "98745"
}
]
}
Shell commands:
import polls.models
polls.models.Order.objects.all() # outputs all orders
table = polls.models.Table()
polls.models.Column(data="doc",table=table)
polls.models.Column(data="order",table=table)
polls.models.Column(data="nothing",table=table)
polls.models.Line(doc="564251422",order="56421",nothing="0.0",table=table)
polls.models.Line(doc="546546545",order="98745",nothing="0.0",table=table)
views.py (update):
bunch = OrderSerializer(Order.objects.all(), many=True)
headers = bunch.data[0].keys()
headers_prepared = map (lambda x: {'data': x} , headers)
ordered_all = ( ('columns', headers_prepared), ('lines', bunch.data) )
data = collections.OrderedDict(ordered_all)
data_json = JSONRenderer().render(data)
return JsonResponse(data_json, safe=False)
As I see this question may relate to another (mentioned in comments) , but for this particular case...
As you are using DjangoRestFramework (the right way), to get all data in Json format you may:
in serialisers.py:
from rest_framework import serializers
from polls.models import Order
class OrderSerializer(serializers.ModelSerializer):
class Meta:
model = Order
fields = ('doc', 'order', 'nothing')
# note we do not use other serializers
next in shell (views in future):
from rest_framework.renderers import JSONRenderer
from polls.models import Order
from polls.serializers import OrderSerializer
bunch = OrderSerializer(Order.objects.all(), many=True)
#this will output "lines" part of desired output
JSONRenderer().render(bunch.data)
#next to get columns, or i say headers
headers = bunch.data[0].keys()
# headers = ['doc','order','nothing']
# !note will fail if bunch.data is empty
headers_prepared = map (lambda x: {'data': x} , headers)
# headers_prepared = [{'data': 'doc'}, {'data': 'order'}, {'data': 'nothing'}]
import collections # need to use OrderedDict to store in our sequence
ordered_all = ( ('columns', headers_prepared), ('lines', bunch.data) )
#finally desired output
JSONRenderer().render( collections.OrderedDict(ordered_all) )
#all code and output tested on my dummy data
UPDATE in urls.py:
urlpatterns = [
...
url(r'my/', my),
...
]
add in views.py:
#as you sileny JSONResponse (from RestFramework docs) - need to show it
from django.http import HttpResponse
class JSONResponse(HttpResponse):
"""
An HttpResponse that renders its content into JSON.
"""
def __init__(self, data, **kwargs):
content = JSONRenderer().render(data)
kwargs['content_type'] = 'application/json'
super(JSONResponse, self).__init__(content, **kwargs)
#now our view
import collections
def my(request):
bunch = OrderSerializer(Order.objects.all(), many=True)
# to get headers independent of presence of orders
empty = OrderSerializer()
headers = empty.data.keys()
headers_prepared = map (lambda x: {'data': x} , headers)
ordered_all = ( ('columns', headers_prepared), ('lines', bunch.data) )
out = collections.OrderedDict(ordered_all)
#finally desired output
return JSONResponse( out )
I've decided to migrate my Django project to work with SqlAlchemy instead of Django ORM and I'm trying to serialize my SqlAlchemy output to a JSON which contains column names.
In Django I had the following code:
logs = Log.objects.values('log_timestamp', 'message', 'exception', 'level__level', 'job_info__job_name', 'machine', 'user', 'job_report__id').filter(job_info__app_id = app_id).order_by('-time_added')[:1]
logs = json.dumps(list(logs), default=views_utils.default_json_serializer)
print(logs)
and here an output example (contains column names):
[{"user": "user", "level__level": "INFO", "message": "this is a message", "log_timestamp": null, "job_info__job_name": "MongoDB_Maintenance", "exception": "exception details", "machine": "machine", "job_report__id": 65}]
Here is my SqlAlchemy code:
res = session.query(func.DATETIME(LogObj.time_added), LogObj.message, LogObj.exception, LogLevelObj.level, LogObj.machine, LogObj.user).\
join(PeriodicJobInfoObj, LogLevelObj, LogObj,aliased=True).\
filter(PeriodicJobInfoObj.app_id == app_id).\
order_by(desc(LogObj.time_added))[:1]
res = json.dumps(res, default=views_utils.default_json_serializer)
print(res)
The output does not contain column names:
[["2015-09-28 15:36:33", "this is a message", "exception details", "CRITICAL", "machine", "user"]]
default_json_serializer code:
def default_json_serializer(obj):
"""Default JSON serializer."""
import calendar, datetime
if isinstance(obj, datetime.datetime):
if obj.utcoffset() is not None:
obj = obj - obj.utcoffset()
millis = int(
calendar.timegm(obj.timetuple()) * 1000 +
obj.microsecond / 1000
)
return millis
How can I achieve similar output like Django does?
(working with Python 3.4.2)
I have managed to do it this way:
def alchemyencoder(obj):
if isinstance(obj, datetime.date):
return obj.isoformat()
elif isinstance(obj, decimal.Decimal):
return float(obj)
I use the above encoder when retrieving records in following way:
row = conn.execute(Model.__table__.select())
d = json.dumps([dict(r) for r in row], default=alchemyencoder)
d = json.JSONDecoder().decode(d)
print(d)