I have json file in my static folder in my django project. And I want to save the data from json file to my Django Database.
This is my model.py
from django.db import models
class Coming_Soon(models.Model):
movie_id = models.CharField(primary_key=True, max_length=11)
movie_title = models.TextField(blank=True, max_length=100)
image_url = models.TextField(blank=True)
synopsis = models.TextField(blank=True)
rating = models.TextField(default="MTRCB rating not yet available")
cast = models.TextField(blank=True)
release_date = models.TextField()
this is sample of my json file
{
"results": [
{
"rating": "PG",
"cast": [
"Jeremy Ray Taylor",
"Gerard Butler",
"Abbie Cornish"
],
"synopsis": "some synopsis",
"movie_title": "(3D/4DX) GEOSTORM",
"image_url": "some url",
"id": "8595"
},
{
"rating": "PG",
"cast": [
"Jeremy Ray Taylor",
"Gerard Butler",
"Abbie Cornish"
],
"synopsis": "some synopsis",
"movie_title": "(ATMOS) GEOSTORM",
"image_url": "some url",
"id": "8604"
}
]
}
what should I write on my view so that I can save those data into my django database? This is my views.py:
def polls(request):
ROOT_FILE = STATIC_ROOT + '/polls/coming_soon.json'
json_data = open(ROOT_FILE)
json_load = json.load(json_data)
with open(ROOT_FILE, 'r') as data:
parsed_json = json.load(data)
for result in parsed_json['results']:
# HOW CAN I SAVE DATA INTO DATABASE??
I really don't have any idea about getting those file into my database. Thank you.
You can create obj like this
from .models import Coming_Soon
def polls(request):
ROOT_FILE = STATIC_ROOT + '/polls/coming_soon.json'
json_data = open(ROOT_FILE)
json_load = json.load(json_data)
with open(ROOT_FILE, 'r') as data:
parsed_json = json.load(data)
for result in parsed_json['results']:
Coming_Soon.objects.create(
movie_id = result['id'],
movie_title = result['movie_title'],
image_url=result['image_url'],
synopsis = result['synopsis'],
)
Related
models.py
class Project(models.Model):
project_id = models.CharField(max_length=50,default=uuid.uuid4, editable=False, unique=True, primary_key=True)
org = models.ForeignKey(Organisation, on_delete=models.CASCADE, related_name='org_project',null=True)
product = models.ManyToManyField(Product,related_name='product_project')
client = models.ForeignKey(Client, on_delete=models.CASCADE, related_name='client_project')
project_name = models.CharField(unique=True,max_length=100)
project_code = models.CharField(max_length=20)
project_cost = models.IntegerField(null=True)
currency_type = models.CharField(max_length=100, choices=CURRENCY_CHOICES, default='Indian Rupee')
project_head = models.ForeignKey(User_Master, on_delete=models.CASCADE, related_name='project_head',null=True)
project_manager = models.ForeignKey(User_Master, on_delete=models.CASCADE, related_name='project_manager',null=True)
project_user = models.ManyToManyField(User_Master,related_name='project_user')
description = models.TextField(max_length=500)
start_date = models.DateField()
end_date = models.DateField()
techstack = models.ManyToManyField(Techstack,related_name='techstack_project')
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
is_active = models.IntegerField(default=0, choices=STATUS_CHOICES)
views.py
from djmoney.settings import CURRENCY_CHOICES
class CurrencyList(APIView):
renderer_classes = (CustomRenderer,)
def get(self, request, *args, **kwargs):
return Response(CURRENCY_CHOICES, status=status.HTTP_200_OK)
I was Trying to get the Currency code list in a response so that the data can be sent to the frontend as a dropdown list.
When I Used the above django money package I am getting an response as
{
"status": "success",
"code": 200,
"data": [
[
"XUA",
"ADB Unit of Account"
],
[
"AFN",
"Afghan Afghani"
],
[
"AFA",
"Afghan Afghani (1927–2002)"
],
....
],
"message": null
}
But I need to Get the response as list of json inside the data not as the list of arrays,
{
"status": "success",
"code": 200,
"data": [
{
"shortname": "XUA",
"fullname": "ADB Unit of Account"
},
{
"shortname":"AFN",
"fullname":"Afghan Afghani"
},
{
"shortname":"AFA",
"fullname":"Afghan Afghani (1927–2002)"
},
....
],
"message": null
}
Is it possible to modify the response for this package? or is there any other way to achieve the end goal response by any other method?
You just need to adjust your response in the following way:
from djmoney.settings import CURRENCY_CHOICES
class CurrencyList(APIView):
renderer_classes = (CustomRenderer,)
def get(self, request, *args, **kwargs):
return Response(
[{'shortname': short, 'fullname': full} for short, full in CURRENCY_CHOICES],
status=status.HTTP_200_OK,
)
try this
from djmoney.settings import CURRENCY_CHOICES
class CurrencyList(APIView):
renderer_classes = (CustomRenderer,)
def get(self, request, *args, **kwargs):
return Response([dict(zip(['shortname', 'fullname'], c)) for c in CURRENCY_CHOICES], status=status.HTTP_200_OK)
python console
>>> data = [
... [
... "XUA",
... "ADB Unit of Account"
... ],
... [
... "AFN",
... "Afghan Afghani"
... ],
... [
... "AFA",
... "Afghan Afghani (1927–2002)"
... ]
]
>>>
>>> [dict(zip(['shortname', 'fullname'], c)) for c in data ]
[{'shortname': 'XUA', 'fullname': 'ADB Unit of Account'}, {'shortname': 'AFN', 'fullname': 'Afghan Afghani'}, {'shortname': 'AFA', 'fullname': 'Afghan Afghani (1927–2002)'}]
I am trying to join a dataframe1 generated by the JSON with dataframe2 using the field order_id, then assign the "status" from dataframe2 to the "status" of dataframe1. Anyone knows how to do this. Many thanks for your help.
dataframe1
[{
"client_id": 1,
"name": "Test01",
"olist": [{
"order_id": 10000,
"order_dt_tm": "2012-12-01",
"status": "" <== use "status" from dataframe2 to populate this field
},
{
"order_id": 10000,
"order_dt_tm": "2012-12-01",
"status": ""
}
]
},
{
"client_id": 2,
"name": "Test02",
"olist": [{
"order_id": 10002,
"order_dt_tm": "2012-12-01",
"status": ""
},
{
"order_id": 10003,
"order_dt_tm": "2012-12-01",
"status": ""
}
]
}
]
dataframe2
order_id status
10002 "Delivered"
10001 "Ordered"
Here is your raw dataset as a json string:
d = """[{
"client_id": 1,
"name": "Test01",
"olist": [{
"order_id": 10000,
"order_dt_tm": "2012-12-01",
"status": ""
},
{
"order_id": 10000,
"order_dt_tm": "2012-12-01",
"status": ""
}
]
},
{
"client_id": 2,
"name": "Test02",
"olist": [{
"order_id": 10002,
"order_dt_tm": "2012-12-01",
"status": ""
},
{
"order_id": 10003,
"order_dt_tm": "2012-12-01",
"status": ""
}
]
}
]"""
Firstly, I would load it as json:
import json
data = json.loads(d)
Then, I would turn it into a Pandas dataframe, notice that I remove status field as it will be populated by the join step :
df1 = pd.json_normalize(data, 'olist')[['order_id', 'order_dt_tm']]
Then, from the second dataframe sample, I would do a left join using merge function:
data = {'order_id':[10002, 10001],'status':['Delivered', 'Ordered']}
df2 = pd.DataFrame(data)
result = df1.merge(df2, on='order_id', how='left')
Good luck
UPDATE
# JSON to Dataframe
df1 = pd.json_normalize(data)
# Sub JSON to dataframe
df1['sub_df'] = df1['olist'].apply(lambda x: pd.json_normalize(x).drop('status', axis=1))
# Build second dataframe
data2 = {'order_id':[10002, 10001],'status':['Delivered', 'Ordered']}
df2 = pd.DataFrame(data2)
# Populates status in sub dataframes
df1['sub_df'] = df1['sub_df'].apply(lambda x: x.merge(df2, on='order_id', how='left').fillna(''))
# Sub dataframes back to JSON
def back_to_json_str(df):
# turns a df back to string json
return str(df.to_json(orient="records", indent=4))
df1['olist'] = df1['sub_df'].apply(lambda x: back_to_json_str(x))
# Global DF back to JSON string
parsed = str(df1.drop('sub_df', axis=1).to_json(orient="records", indent=4))
parsed = parsed.replace(r'\n', '\n')
parsed = parsed.replace(r'\"', '\"')
# Print result
print(parsed)
UPDATE 2
here is a way to add index colum to a dataframe:
df1['index'] = [e for e in range(df1.shape[0])]
This is my code assigning title values from a dataframe back to the JSON object. The assignment operation takes a bit time if the number records in the JSON object is 100000. Anyone knows how to improve the performance of this code. Many thanks.
import json
import random
import pandas as pd
import pydash as _
data = [{"pid":1,"name":"Test1","title":""},{"pid":2,"name":"Test2","title":""}] # 5000 records
# dataframe1
df = pd.json_normalize(data)
# dataframe2
pid = [x for x in range(1, 5000)]
title_set = ["Boss", "CEO", "CFO", "PMO", "Team Lead"]
titles = [title_set[random.randrange(0, 5)] for x in range(1, 5000)]
df2 = pd.DataFrame({'pid': pid, 'title': titles})
#left join dataframe1 and dataframe2
df3 = df.merge(df2, on='pid', how='left')
#assign title values from dataframe back to the json object
for row in df3.iterrows():
idx = _.find_index(data, lambda x: x['pid'] == row[1]['pid'])
data[idx]['title'] = row[1]['title_y']
print(data)
I have a Django REST API endpoint with this structure, that I need to post to an external API:
{
"body": [
"...",
"...",
"...",
],
"title": [
"...",
"...",
"...",
],
"id": [
"...",
"...",
"...",
]
}
The first item under 'body' goes with the first under 'title' and 'id', and so forth.
The problem I'm having is that the API in question expects JSON data with the following structure:
{
"texts": [
{
"body": "...",
"title": "...",
"id": "..."
},
{
"body": "...",
"title": "...",
"id": "..."
},
{
"body": "...",
"title": "...",
"id": "..."
},
],
"language": "EN",
}
And I can't figure out how have my endpoint mirror that structure, with the bodies, titles, and ids grouped together, those groupings nested under texts, and with the language parameter appended at the end.
The serializer I'm using in my views.py looks as follows:
class MyReasonsSerializer(serializers.Serializer):
body = serializers.SerializerMethodField()
title = serializers.SerializerMethodField()
id = serializers.SerializerMethodField()
def get_body(self, obj):
return obj.reasons.order_by('transaction_date').values_list('body', flat=True)
def get_title(self, obj):
return obj.reasons.order_by('transaction_date').values_list('title', flat=True)
def get_id(self, obj):
return obj.reasons.order_by('transaction_date').values_list('transaction_date', flat=True)
class ReasonsData(RetrieveAPIView):
queryset = Market.objects.all().prefetch_related('reasons')
authentication_classes = []
permission_classes = []
serializer_class = MyReasonsSerializer
Thanks in advance for any advice!
EDIT:
Here are the models:
class Market(models.Model):
title = models.CharField(max_length=50, default="")
current_price = models.DecimalField(max_digits=5, decimal_places=2, default=0.50)
description = models.TextField(default="")
...
language = models.CharField(max_length=2, default="EN")
def __str__(self):
return self.title[:50]
class Reason(models.Model):
user_id = models.ForeignKey('users.CustomUser',
on_delete=models.CASCADE,
related_name='user_reasons',
default=None)
market = models.ForeignKey(
Market,
on_delete=models.CASCADE,
related_name='reasons',
default=None)
...
valence = models.CharField(max_length=11, default="")
title = models.TextField(default="")
body = models.TextField(default="")
def __str__(self):
return str(self.title)
I would structure it like this... (its hard to know exactly without seeing the models and trying it out)
class ReasonSerializer(serializers.ModelSerializer):
class Meta:
model = Reason
fields = ("id", "body", "title")
class MarketSerializer(serializers.Serializer):
texts = ReasonSerializer(many=True, source="reasons")
language = serializers.CharField()
class Meta:
model = Market
class ReasonsData(RetrieveAPIView):
queryset = Market.objects.all()
authentication_classes = []
permission_classes = []
serializer_class = MarketSerializer
def get_queryset(self):
qs = super().get_queryset()
# filter queryset by valence if passed as a query param
# ?valence=something
valence = self.request.query_params.get("valence", None)
if valence is not None:
qs = qs.filter(reasons__valence=valence)
return qs
I am new to Django and Django-REST and I was tasked to create a serializer.
The output of my serializer looks like this:
{
"count": 1,
"next": null,
"previous": null,
"results": [
{
"id": 1,
"name": "a",
"description": "a",
"price": "1.00",
"images": [
{
"image_addr": "http://127.0.0.1:8000/media/products/Screenshot_from_2018-05-16_15-07-34.png",
"product": 1
},
{
"image_addr": "http://127.0.0.1:8000/media/products/Screenshot_from_2018-05-16_16-42-55.png",
"product": 1
}
]
}
]
}
How can i tweak my serializer in a way that my output would look like this:
{
"count": 1,
"next": null,
"previous": null,
"results": [
{
"id": 1,
"name": "a",
"description": "a",
"price": "1.00",
"images": [
"http://127.0.0.1:8000/media/products/Screenshot_from_2018-05-16_15-07-34.png",
"http://127.0.0.1:8000/media/products/Screenshot_from_2018-05-16_16-42-55.png"
]
}
]
}
The serializers that I am using are:
class ProductImageSerializer(serializers.ModelSerializer):
class Meta:
model = ProductImage
fields = ('image_addr', 'product')
and
class ProductSerializer(serializers.ModelSerializer):
images = ProductImageSerializer(many=True, read_only=True)
class Meta:
model = Product
fields = ('id', 'name', 'description', 'price','images')
My models used are:
class ProductImage(models.Model):
image_addr = models.FileField(upload_to='products/',null=True)
product = models.ForeignKey(Product, on_delete=models.CASCADE, related_name='images')
and
class Product(models.Model):
name = models.CharField(max_length=150)
price = models.DecimalField(max_digits=9, decimal_places=2)
product_count = models.IntegerField(blank=True, default=0, validators=[MinValueValidator(0)])
description = models.TextField()
category = models.ManyToManyField(Category)
objects = ProductManager()
My view used is:
class CategoryProductList(generics.ListAPIView):
serializer_class = ProductSerializer
def get_queryset(self):
queryset = Product.objects.filter(category__id=self.kwargs['pk'])
return queryset
You can use SerializerMethodField for your purpose. Change your ProductSerializer as below
class ProductSerializer(serializers.ModelSerializer):
images = serializers.SerializerMethodField(read_only=True, source='images')
def get_images(self, model):
return model.images.values_list('image_addr', flat=True)
class Meta:
model = Product
fields = ('id', 'name', 'description', 'price', 'images')
Lets say we have our MongoDB and we want to backup the data into a .json file example for an output file : database.json and inside :
{
"collections": [
{"name": "admin"},
{"name": "class"},
{"name": "lesson"},
{"name": "message"},
{"name": "room"},
{"name": "student"},
{"name": "subject"},
{"name": "teacher"}
],
"subjects": [
{
"name": "Null",
"color": "#FFFFFF"
},
{
"name": "Design Art",
"color": "#82B9D6"
},
{
"name": "Plastic Art",
"color": "#a3db05"
},
{
"name": "Media And Production",
"color": "#522a64"
}, //...there is a continue to this file ....
}
each collection should be added to the collections and for each collection there should be an array of all the info inside it (like above)
I'm using python 3.4 with the pymongo driver.
What is the best way to get all the info from the DB , create the JSON object and insert it to a new .json file
I found this way of doing it :
import json
from pymongo import MongoClient
from pprint import pprint
def main():
with open('../config/database.json') as database_config:
config = json.load(database_config)
client = MongoClient(config["mongodb"])
db = client[config["database"]]
data = dict()
data["collections"] = db.collection_names()
for collection_name in db.collection_names():
data[collection_name] = get_collection(db, collection_name)
pprint(data)
insert_data_to_file(data)
def get_collection(db, collection_name):
collection_list = []
collection = db[collection_name]
cursor = collection.find({})
for document in cursor:
_id = document.pop('_id')
document['_id'] = str(_id)
collection_list.append(document)
return collection_list
def insert_data_to_file(data):
with open('database.json', 'x') as database:
json.dump(data, database, sort_keys=True)
main()