I currently have two classes in Python like these ones
class person:
age=""
name=""
ranking = {}
def addRanking():
#Do Whatever treatment and add to the ranking dict
class ranking:
semester = ""
position = ""
gpa = ""
I have my list of person as a dictionary called dictP json.dumps() this dictionary but it seems that it doesn't work. Here is my function to dump to JSON
def toJson():
jsonfile = open('dict.json', 'w')
print(json.dump(listP, jsonfile))
I get the famous: is not JSON serializable.
Would you know what I can do to help this problem. I thought that having two dictionaries (which are serializable) would avoid this kind of issue, but apparently not.
Thanks in advance
Edit:
Here is an example (typed on my phone sorry for typos, I'm not sure it does run but it's so you get the idea):
class person:
age=""
name=""
ranking = {}
def __init__(self, age, name):
self.age = age
self.name = name
self.ranking = {}
def addRanking(self,semester,position,gpa):
#if the semester is not already present in the data for that person
self.ranking[semester] = make_ranking(semester,position,gpa)
class ranking:
semester = ""
position = ""
gpa = ""
def __init__(self, semester, position, gpa):
self.semester = semester
self.position = position
self.gpa = gpa
dictP = {}
def make_person(age, name):
# Some stuff happens there
return person(age,name)
def make_ranking(semester,postion,gpa):
#some computation there
return ranking(semester,position,gpa)
def pretending_to_read_csv():
age = 12
name = "Alice"
p = make_person(age, name)
dictP["1"] = p
age = 13
name = "Alice"
p = make_person(age, name)
dictP["2"] = p
#We read a csv for ranking that gives us an ID
semester = 1
position = 4
gpa = 3.2
id = 1
dictP["1"].addRanking(semester, position, gpa)
semester = 2
position = 4
gpa = 3.2
id = 1
dictP["1"].addRanking(semester, position, gpa)
For a dictionary to be serializable, note that all the keys & values in that dictionary must be serializable as well. You did not show us what listP contains, but I'm guessing it's something like this:
>>> listP
[<__main__.person instance at 0x107b65290>, <__main__.person instance at 0x107b65368>]
Python instances are not serializable.
I think you want a list of dictionaries, which would look like this:
>>> listP
[{'ranking': {}, 'age': 10, 'name': 'fred'}, {'ranking': {}, 'age': 20, 'name': 'mary'}]
This would serialize as you expect:
>>> import json
>>> json.dumps(listP)
'[{"ranking": {}, "age": 10, "name": "fred"}, {"ranking": {}, "age": 20, "name": "mary"}]'
UPDATE
(Thanks for adding example code.)
>>> pretending_to_read_csv()
>>> dictP
{'1': <__main__.person instance at 0x107b65368>, '2': <__main__.person instance at 0x107b863b0>}
Recall that user-defined classes cannot be serialized automatically. It's possible to extend the JSONEncoder directly to handle these cases, but all you really need is a function that can turn your object into a dictionary comprised entirely of primitives.
def convert_ranking(ranking):
return {
"semester": ranking.semester,
"position": ranking.position,
"gpa": ranking.gpa}
def convert_person(person):
return {
"age": person.age,
"name": person.name,
"ranking": {semester: convert_ranking(ranking) for semester, ranking in person.ranking.iteritems()}}
One more dictionary comprehension to actually do the conversion and you're all set:
>>> new_dict = {person_id: convert_person(person) for person_id, person in dictP.iteritems()}
>>> from pprint import pprint
>>> pprint(new_dict)
{'1': {'age': 12,
'name': 'Alice',
'ranking': {1: {'gpa': 3.2, 'position': 4, 'semester': 1},
2: {'gpa': 3.2, 'position': 4, 'semester': 2}}},
'2': {'age': 13, 'name': 'Alice', 'ranking': {}}}
Since no user-defined objects are stuffed in there, this will serialize as you hope:
>>> json.dumps(new_dict)
'{"1": {"ranking": {"1": {"position": 4, "semester": 1, "gpa": 3.2}, "2": {"position": 4, "semester": 2, "gpa": 3.2}}, "age": 12, "name": "Alice"}, "2": {"ranking": {}, "age": 13, "name": "Alice"}}'
You can try calling json.dump on the .__dict__ member of your instance. You say that you have a list of person instances so try doing something like this:
listJSON = []
for p in listP
#append the value of the dictionary containing data about your person instance to a list
listJSON.append(p.__dict__)
json.dump(listJSON, jsonfile)
If you are storing your person instances in a dictionary like so: dictP = {'person1': p1, 'person2': p2} this solution will loop through the keys and change their corresponding values to the __dict__ member of the instance:
for key in dictP:
dictP[key] = dictP[key].__dict__
json.dump(dictP, jsonfile)
Related
I'm writing a simple program in Python that iterates through various API JSON links, and compares values from each of the data sources. As I cycle through one of the JSON files, I call a function that calls a different JSON file based off of one of the values of the initial JSON file.
Here is my Python code:
import requests
import json
import urllib.request as request
from urllib.request import Request, urlopen
import time
# THINGS
'''
international API: https://arsonwarehouse.com/api/v1/foreign-stock
domestic API: https://api.torn.com/market/ ?selections=itemmarket&key=KEY
personal API: https://api.torn.com/user/2519609?selections=money&key=KEY
'''
capacity = 21
moneyOverride = False
totalCost: int
totalIncome: int
#Takes in the cost per item, multiplies by capacity, returns true if it's affordable
def checkAfford(costPer):
global totalCost
totalCost = capacity * costPer
if totalCost < money:
return True
else:
return False
#Puts together the API link with the ID number and calls pullJSON()
def checkDomestic(id):
jsonPULL = pullJSON(f"https://api.torn.com/market/{id}?selections=itemmarket&key=YET")
if jsonPULL == None:
print("error")
else:
return jsonPULL
#Requests the JSON from the server
def pullJSON(url):
req = Request(url, headers={'User-Agent': 'Mozilla/5.0'})
webpage = urlopen(req).read()
with urlopen(req) as response:
if response.getcode() == 200:
source = response.read()
data = json.loads(source)
return data
else:
print('An error occurred while attempting to retrieve data from the API.')
internationalData = pullJSON("https://arsonwarehouse.com/api/v1/foreign-stock")
personalData = pullJSON("https://api.torn.com/user/2519609?selections=money&key=YET")
domesticData = checkDomestic(10)
#Money override
if moneyOverride == False:
money = personalData.get("money_onhand")
else:
money = 1000000000000000
print("onhand:", money)
i = 0
length = len(internationalData["items"])
print(length)
for x in internationalData["items"]:
time.sleep(0.5)
ident = x["item_id"]
domPrice = x["price"]
print("DOMESTIC: ID:", ident, " at price:", domPrice)
domestic = checkDomestic(ident).get("itemmarket")[1]
inPrice = domestic["cost"]
if inPrice == None:
print("error", domestic["ID"])
location = x["country"]
print("INTERNATIONAL: ID:", ident, " at price:", inPrice, "location: ", location)
print("end")
This code won't inherently work — I removed my API key from the links. However, I can show what the example JSON looks like.
This is what checkDomestic() pulls from:
{
"itemmarket": [
{
"ID": 101242786,
"cost": 17898,
"quantity": 1
},
{
"ID": 101242784,
"cost": 17898,
"quantity": 1
}
]
}
And this is what internationalData looks like:
{
"items": [
{
"item_id": 4,
"country": "South Africa",
"price": 750,
"stock": 247,
"reported_at": "2020-06-03T11:47:56Z"
},
{
"item_id": 8,
"country": "Mexico",
"price": 4200,
"stock": 59,
"reported_at": "2020-06-03T11:45:21Z"
}
]
}
I've searched around for an answer, but most questions have to do with the .sort() method. I should also mention that this happens at random—sometimes in the first 10 iterations, but never at the end, so it's not an out of range error. The JSON structures stay the same across.
A somewhat related question: I've completed this project before in Golang, and I noticed that in Go it finishes much, much, much faster than in Python. Is there some kind of redundancy that's making it take so long? I'm aware of the time.sleep() function I have in there-it was to make sure that it's not something with overcalling the API.
Any advice would be massively helpful!
I am trying to deserialise a json string to an object using jsons but having problems with nested objects, but can't work out the syntax.
As an example the following code attempts to define the data structure as a series of dataclasses but fails to deserialise the nested objects C and D ? The syntax is clearly wrong, but its not clear to me how it should structured
import jsons
from dataclasses import dataclass
#dataclass
class D:
E: str
class C:
id: int
name:str
#dataclass
class test:
A: str
B: int
C: C()
D: D()
jsonString = {"A":"a","B":1,"C":[{"id":1,"name":"one"},{"id":2,"name":"two"}],"D":[{"E":"e"}]}
instance = jsons.load(jsonString, test)
Can anyone indicate the correct way to deserialise the objects from json ?
There are two relatively simple problems with your attempt:
You forgot to decorate C with #dataclass.
Test.C and Test.D aren't defined with types, but with instances of the types. (Further, you want both fields to be lists of the given type, not single instances of each.)
Given the code
import jsons
from dataclasses import dataclass
from typing import List
#dataclass
class D:
E: str
#dataclass # Problem 1 fixed
class C:
id: int
name: str
#dataclass
class Test:
A: str
B: int
C: List[C] # Problem 2 fixed; List[C] not C() or even C
D: List[D] # Problem 2 fixed; List[D], not D() or even D
Then
>>> obj = {"A":"a", "B":1, "C": [{"id": 1,"name": "one"}, {"id": 2, "name": "two"}], "D":[{"E": "e"}]}
>>> jsons.load(obj, Test)
test(A='a', B=1, C=[C(id=1, name='one'), C(id=2, name='two')], D=[D(E='e')])
from dataclasses import dataclass
from typing import List
from validated_dc import ValidatedDC
#dataclass
class D(ValidatedDC):
E: str
#dataclass
class C(ValidatedDC):
id: int
name: str
#dataclass
class Test(ValidatedDC):
A: str
B: int
C: List[C]
D: List[D]
jsonString = {
"A": "a",
"B": 1,
"C": [{"id": 1, "name": "one"}, {"id": 2, "name": "two"}],
"D": [{"E": "e"}]
}
instance = Test(**jsonString)
assert instance.C == [C(id=1, name='one'), C(id=2, name='two')]
assert instance.C[0].id == 1
assert instance.C[1].name == 'two'
assert instance.D == [D(E='e')]
assert instance.D[0].E == 'e'
ValidatedDC: https://github.com/EvgeniyBurdin/validated_dc
You can do something like this:
from collections import namedtuple
# First parameter is the class/tuple name, second parameter
# is a space delimited string of varaibles.
# Note that the variable names should match the keys from
# your dictionary of arguments unless only one argument is given.
A = namedtuple("A", "a_val") # Here the argument `a_val` can be called something else
B = namedtuple("B", "num")
C = namedtuple("C", "id name")
D = namedtuple("D", "E") # This must be `E` since E is the key in the dictionary.
# If you dont want immutable objects to can use full classes
# instead of namedtuples
# A dictionary which matches the name of an object seen in a payload
# to the object we want to create for that name.
object_options = {
"A": A,
"B": B,
"C": C,
"D": D
}
my_objects = [] # This is the list of object we get from the payload
jsonString = {"A":"a","B":1,"C":[{"id":1,"name":"one"},{"id":2,"name":"two"}],"D":[{"E":"e"}]}
for key, val in jsonString.items():
if key in object_options: # If this is a valid object
if isinstance(val, list): # If it is a list of this object
for v in val: # Then we need to add each object in the list
my_objects.append(object_options[key](**v))
elif isinstance(val, dict): # If the object requires a dict then pass the whole dict as arugments
my_objects.append(object_options[key](**val))
else: # Else just add this object with a singular argument.
my_objects.append(object_options[key](val))
print(my_objects)
Output:
[A(a_val='a'), B(num=1), C(id=1, name='one'), C(id=2, name='two'), D(E='e')]
I've finally managed to get this to work by removing the dataClass definition and expanding the class definitions old school.... code as follows...
import jsons
class D:
def __init__(self, E = ""):
self.E = E
class C:
def __init__(self, id = 0, name=""):
self.id = id
self.name = name
class test:
def __init__(self, A = "", B = 0, C = C(), D = D()):
self.A = A
self.B = B
self.C = C
self.D = D
jsonString = {"A":"a","B":1,"C":[{"id":1,"name":"one"},{"id":2,"name":"two"}],"D":[{"E":"e"}]}
instance = jsons.load(jsonString, test)
It now works but is not as clean as with a dataClass. Grateful if anyone can indicate how the original post can be constructed with the dataClass definition.
I want to load below json data in my Model.
{
"99popularity": 79.0,
"director": "William Cottrell",
"genre": [
"Animation",
" Family",
" Fantasy",
" Musical",
" Romance"
],
"imdb_score": 7.9,
"name": "Snow White and the Seven Dwarfs"
},
{
"99popularity": 84.0,
"director": "Stanley Kubrick",
"genre": [
"Adventure",
" Mystery",
" Sci-Fi"
],
"imdb_score": 8.4,
"name": "2001 : A Space Odyssey"
},
I have create two models using reference to json data
class Genre(models.Model):
name = models.CharField(max_length=30)
class Movie(models.Model):
popularity = models.FloatField(max_length=10)
director = models.CharField(max_length=30)
genre = models.ManyToManyField(Genre)
imdb_score = models.FloatField(max_length=10)
name = models.CharField(max_length=30)
But in Genre Model i don't have any data and in json in genre section their is no id instead name. How can i load that data in my model. Please Help.
You can use the get_or_create method, but you have to make the name field unique. To create two models using reference to json data, I would use a custom class method like this:
class Genre(models.Model):
name = models.CharField(max_length=30, unique=True) # make unique
class Movie(models.Model):
popularity = models.FloatField(max_length=10)
director = models.CharField(max_length=30)
genre = models.ManyToManyField(Genre)
imdb_score = models.FloatField(max_length=10)
name = models.CharField(max_length=30)
#classmethod
def create(cls, **kwargs):
movie = cls.objects.create(
popularity=kwargs['99popularity'],
director=kwargs['director'],
imdb_score=kwargs['imdb_score'],
name=kwargs['name']
)
for genre_name in kwargs['genre']:
genre, created = Genre.objects.get_or_create(name=genre_name)
movie.genre.add(genre)
return movie
Assuming you converted your json data to a string you can do this:
import json
from .models import Movie
# you can also keep this inside a view
with open('movie_data.json', encoding='utf-8') as data_file:
json_data = json.loads(data_file.read())
for movie_data in json_data:
movie = Movie.create(**movie_data)
# movie and genres created
How do I send a Django model and it's related (_set)s via Ajax?
class Zoo(models.Model):
id = models.AutoField(primary_key=True)
details = models.TextField(blank=True) # HUGE AMOUNT OF DATA
animals_json = models.TextField(blank=True) # Information on what animals are in Zoo
class Animals(models.Model):
id = models.AutoField(primary_key=True)
zoo = models.ForeignKey(Zoo, on_delete=models.PROTECT)
animal_name = models.TextField(blank=True)
zoo = Zoo.objects.filter(pk=5)
animals = []
for animal in json.loads(zoo.animals_json):
animals.append(Animals(zoo=zoo, animal_name=animal['name']))
# Now I have a Zoo model and a list of animals.
# My Zoo model has a animal_set (RelatedManager) object
Now, I would like to send a JSON encoded Zoo via Ajax to front end. How do I do this? Option 1)
zoo_json = serializers.serialize('json', [zoo])
return HttpResposne(zoo_json)
This won't work because then I'm not sending any information about the animals. (Yes I realized it's encoded in the animals_json, but I want to just send clean objects via AJAX and not parse it with a front end loop). Something like this seems very messy as well because now the objects are seperate from one another:
zoo_json = serializers.serialize('json', [zoo])
animals_json = serializers.serialize('json', animals)
data_to_send_via_ajaz = {'zoo' : zoo_json, 'animals' : animals_json}
return HttpResposne(zoo_json)
My ideal object looks like this:
zoo_model_in_json_form [
animal_model_in_json_form,
animal_model_in_json_form,
animal_model_in_json_form,
...
]
return HttpResposne(zoo_model_in_json_form)
This way I can loop through all the animals very easily. Any ideas?
You can always use your own method of serializing the objects using dict objects and finally using json.dumps to dump the dict object as a string with response content_type set to application/json.
Solution:
import json
zoo = Zoo.objects.filter(pk=5).first() # adding `first` call to get a single zoo
serialized_zoo_data = dict()
# serialize zoo with its animals nested in it
# Serialized zoo with animals:
# {
# "id": 1,
# "details": "blah blah",
# "animals": [{
# "id": 222,
# "name": "Gorilla"
# }, {
# "id": 11,
# "name": "Camel"
# }]
# }
if zoo is not None:
# get all animals belonging to a `Zoo` object
zoo_animals = zoo.animal_set.all()
serialized_zoo_data["id"] = zoo.id
serialized_zoo_data["details"] = zoo.details
serialized_zoo_data["animals"] = []
for animal in zoo_animals:
animal = dict()
animal["id"] = animal.id
animal["name"] = animal.animal_name
serialized_zoo_data["animals"].append(animal)
return HttpResponse(json.dumps(serialized_zoo_data), content_type="application/json")
I'm working with nested JSON-like data structures in python 2.7 that I exchange with some foreign perl code. I just want to 'work with' these nested structures of lists and dictionaries in amore pythonic way.
So if I have a structure like this...
a = {
'x': 4,
'y': [2, 3, { 'a': 55, 'b': 66 }],
}
...I want to be able to deal with it in a python script as if it was nested python classes/Structs, like this:
>>> aa = j2p(a) # <<- this is what I'm after.
>>> print aa.x
4
>>> aa.z = 99
>>> print a
{
'x': 4,
'y': [2, 3, { 'a': 55, 'b': 66 }],
'z': 99
}
>>> aa.y[2].b = 999
>>> print a
{
'x': 4,
'y': [2, 3, { 'a': 55, 'b': 999 }],
'z': 99
}
Thus aa is a proxy into the original structure. This is what I came up with so far, inspired by the excellent What is a metaclass in Python? question.
def j2p(x):
"""j2p creates a pythonic interface to nested arrays and
dictionaries, as returned by json readers.
>>> a = { 'x':[5,8], 'y':5}
>>> aa = j2p(a)
>>> aa.y=7
>>> print a
{'x': [5, 8], 'y':7}
>>> aa.x[1]=99
>>> print a
{'x': [5, 99], 'y':7}
>>> aa.x[0] = {'g':5, 'h':9}
>>> print a
{'x': [ {'g':5, 'h':9} , 99], 'y':7}
>>> print aa.x[0].g
5
"""
if isinstance(x, list):
return _list_proxy(x)
elif isinstance(x, dict):
return _dict_proxy(x)
else:
return x
class _list_proxy(object):
def __init__(self, proxied_list):
object.__setattr__(self, 'data', proxied_list)
def __getitem__(self, a):
return j2p(object.__getattribute__(self, 'data').__getitem__(a))
def __setitem__(self, a, v):
return object.__getattribute__(self, 'data').__setitem__(a, v)
class _dict_proxy(_list_proxy):
def __init__(self, proxied_dict):
_list_proxy.__init__(self, proxied_dict)
def __getattribute__(self, a):
return j2p(object.__getattribute__(self, 'data').__getitem__(a))
def __setattr__(self, a, v):
return object.__getattribute__(self, 'data').__setitem__(a, v)
def p2j(x):
"""p2j gives back the underlying json-ic json-ic nested
dictionary/list structure of an object or attribute created with
j2p.
"""
if isinstance(x, (_list_proxy, _dict_proxy)):
return object.__getattribute__(x, 'data')
else:
return x
Now I wonder whether there is an elegant way of mapping a whole set of the __*__ special functions, like __iter__, __delitem__? so I don't need to unwrap things using p2j() just to iterate or do other pythonic stuff.
# today:
for i in p2j(aa.y):
print i
# would like to...
for i in aa.y:
print i
I think you're making this more complex than it needs to be. If I understand you correctly, all you should need to do is this:
import json
class Struct(dict):
def __getattr__(self, name):
return self[name]
def __setattr__(self, name, value):
self[name] = value
def __delattr__(self, name):
del self[name]
j = '{"y": [2, 3, {"a": 55, "b": 66}], "x": 4}'
aa = json.loads(j, object_hook=Struct)
for i in aa.y:
print(i)
When you load JSON, the object_hook parameter lets you specify a callable object to process objects that it loads. I've just used it to turn the dict into an object that allows attribute access to its keys. Docs
There is an attrdict library that does exactly that in a very safe manner, but if you want, a quick and dirty (possibly leaking memory) approach was given in this answer:
class AttrDict(dict):
def __init__(self, *args, **kwargs):
super(AttrDict, self).__init__(*args, **kwargs)
self.__dict__ = self
j = '{"y": [2, 3, {"a": 55, "b": 66}], "x": 4}'
aa = json.loads(j, object_hook=AttrDict)
I found the answer: There is intentionally no way to automatically map the special methods in python, using __getattribute__. So to achieve what I want, I need to explicitely define all special methods like __len__ one after the other.