How can I use ujson as a Flask encoder/decoder? - json

I have seen that once can use simplejson as a JSON encoder / decoder within a Flask application like this:
from simplejson import JSONEncoder, JSONDecoder
app.json_encoder = JSONEncoder
app.json_decoder = JSONDecoder
But ujson does not have such objects:
>>> from ujson import JSONEncoder
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
ImportError: cannot import name 'JSONEncoder' from 'ujson' (/.../site-packages/ujson.cpython-38-x86_64-linux-gnu.so
What I tried
I thought of something like this:
from uuid import UUID, uuid4
import ujson as json
from flask import Flask, jsonify
from flask.json import JSONEncoder
class CustomJSONEncoder(JSONEncoder):
def default(self, obj):
if isinstance(obj, UUID):
return str(obj)
return JSONEncoder.default(self, obj)
def encode(self, o):
return json.dumps(o)
app = Flask(__name__)
app.json_encoder = CustomJSONEncoder
#app.route("/")
def index():
return jsonify({"foo": uuid4()})
app.run()
But I'm uncertain because the help to the decoder shows this:
| decode(self, s, _w=<built-in method match of re.Pattern object at 0x7f6a608404b0>, _PY3=True)
| Return the Python representation of ``s`` (a ``str`` or ``unicode``
| instance containing a JSON document)
|
| raw_decode(self, s, idx=0, _w=<built-in method match of re.Pattern object at 0x7f6a608404b0>, _PY3=True)
| Decode a JSON document from ``s`` (a ``str`` or ``unicode``
| beginning with a JSON document) and return a 2-tuple of the Python
| representation and the index in ``s`` where the document ended.
| Optionally, ``idx`` can be used to specify an offset in ``s`` where
| the JSON document begins.
|
| This can be used to decode a JSON document from a string that may
| have extraneous data at the end.
Is my implementation ok? How would I support those other parameters? When is decode and when raw_decode used?
When I run this, I get:
[2020-10-09 10:54:52,063] ERROR in app: Exception on / [GET]
Traceback (most recent call last):
File "/home/moose/.pyenv/versions/3.8.3/lib/python3.8/site-packages/flask/app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "/home/moose/.pyenv/versions/3.8.3/lib/python3.8/site-packages/flask/app.py", line 1952, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/home/moose/.pyenv/versions/3.8.3/lib/python3.8/site-packages/flask/app.py", line 1821, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/home/moose/.pyenv/versions/3.8.3/lib/python3.8/site-packages/flask/_compat.py", line 39, in reraise
raise value
File "/home/moose/.pyenv/versions/3.8.3/lib/python3.8/site-packages/flask/app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "/home/moose/.pyenv/versions/3.8.3/lib/python3.8/site-packages/flask/app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "main.py", line 28, in index
return jsonify({"foo": uuid4()})
File "/home/moose/.pyenv/versions/3.8.3/lib/python3.8/site-packages/flask/json/__init__.py", line 370, in jsonify
dumps(data, indent=indent, separators=separators) + "\n",
File "/home/moose/.pyenv/versions/3.8.3/lib/python3.8/site-packages/flask/json/__init__.py", line 211, in dumps
rv = _json.dumps(obj, **kwargs)
File "/home/moose/.pyenv/versions/3.8.3/lib/python3.8/site-packages/simplejson/__init__.py", line 398, in dumps
return cls(
File "main.py", line 14, in encode
return json.dumps(o)
TypeError: UUID('1f45a2bc-c964-48f0-b2f5-9ef7a2557966') is not JSON serializable

You can use a try block like that:
import ujson as json
from flask.json import JSONEncoder
class CustomJSONEncoder(JSONEncoder):
def default(self, obj):
try:
return json.dumps(obj)
except TypeError:
return JSONEncoder.default(self, obj)
from flask import Flask
app = Flask(__name__)
app.json_encoder = CustomJSONEncoder

Related

Dash Plotly error TypeError: Object of type DataFrame is not JSON serializable

Hello I am working with Dash for making dashboard.
Below is my code.
I tried to fix the error but not able to fix, Can anyone look into this?
on chrome i am getting. Error loading layout
I am getting TypeError
import dash_bootstrap_components as dbc
from dash import dcc
import dash_html_components as html
from dash import dash_table
import pandas as pd
import numpy as np
def getData():
return preprocess()
def back_to_df(dictio):
return pd.DataFrame.from_dict(dictio)
tblcols =[{"name": i, "id": i} for i in back_to_df(getData()).columns]
app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
body = html.Div([
html.H1("Live rates")
, dbc.Row([
dbc.Col(html.Div([dcc.Interval('graph-update', interval = 80, n_intervals = 0),
dash_table.DataTable(
id = 'table',
data = getData(),
columns=tblcols,
page_size= 10,
style_table={'overflowX': 'auto'},
)]),width=3)
])
])
app.layout = html.Div([body])
#app.callback(
dash.dependencies.Output('table','data'),
[dash.dependencies.Input('graph-update', 'n_intervals')])
def updateTable(n):
return getData()
if __name__ == "__main__":
app.run_server(debug = False, port = 8010)
I tried to fix the error but not able to fix, Can anyone look into this?
I am getting error as follows.
Looking for help for below error. dash pandas plotly dataframe
Traceback (most recent call last):
File "C:\Users\Admin\anaconda3\lib\site-packages\flask\app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "C:\Users\Admin\anaconda3\lib\site-packages\flask\app.py", line 1952, in full_dispatch_request
rv = self.handle_user_exception(e)
File "C:\Users\Admin\anaconda3\lib\site-packages\flask\app.py", line 1821, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "C:\Users\Admin\anaconda3\lib\site-packages\flask\_compat.py", line 39, in reraise
raise value
File "C:\Users\Admin\anaconda3\lib\site-packages\flask\app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "C:\Users\Admin\anaconda3\lib\site-packages\flask\app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "C:\Users\Admin\anaconda3\lib\site-packages\dash\dash.py", line 569, in serve_layout
to_json(layout),
File "C:\Users\Admin\anaconda3\lib\site-packages\dash\_utils.py", line 20, in to_json
return to_json_plotly(value)
File "C:\Users\Admin\anaconda3\lib\site-packages\plotly\io\_json.py", line 124, in to_json_plotly
return json.dumps(plotly_object, cls=PlotlyJSONEncoder, **opts)
File "C:\Users\Admin\anaconda3\lib\json\__init__.py", line 234, in dumps
return cls(
File "C:\Users\Admin\anaconda3\lib\site-packages\_plotly_utils\utils.py", line 59, in encode
encoded_o = super(PlotlyJSONEncoder, self).encode(o)
File "C:\Users\Admin\anaconda3\lib\json\encoder.py", line 199, in encode
chunks = self.iterencode(o, _one_shot=True)
File "C:\Users\Admin\anaconda3\lib\json\encoder.py", line 257, in iterencode
return _iterencode(o, 0)
File "C:\Users\Admin\anaconda3\lib\site-packages\_plotly_utils\utils.py", line 136, in default
return _json.JSONEncoder.default(self, obj)
File "C:\Users\Admin\anaconda3\lib\json\encoder.py", line 179, in default
raise TypeError(f'Object of type {o.__class__.__name__} '
TypeError: Object of type DataFrame is not JSON serializable```
Sounds like the getData function is returning a pandas DataFrame directly. That won't work. You'll need to do this:
return df.to_dict(orient='records')
That should work.

ValueError: arrays must all be same length - Parse the JSON into Pandas DataFrame

import requests
import json
import pandas as pd
data = requests.get("https://earthquake.usgs.gov/fdsnws/event/1/query?format=geojson")
json_data = data.json()
with open(r"C:\path\file.json",'w') as outfile:
json.dump(json_data, outfile)
df = pd.read_json(r"C:\path\file.json")
when I tried to parse the json data into Pandas Dataframe, I get the below error:
ValueError: arrays must all be same length
Can anyone help me out in this?
Traceback (most recent call last):
File "c:/path/file.py", line 29, in <module>
df = pd.read_json(r"C:\path\file.json")
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\util\_decorators.py", line 199, in wrapper
return func(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\util\_decorators.py", line 296, in wrapper
return func(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\io\json\_json.py", line 618, in read_json
result = json_reader.read()
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\io\json\_json.py", line 755, in read
obj = self._get_object_parser(self.data)
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\io\json\_json.py", line 777, in _get_object_parser
obj = FrameParser(json, **kwargs).parse()
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\io\json\_json.py", line 886, in parse
self._parse_no_numpy()
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\io\json\_json.py", line 1118, in _parse_no_numpy
self.obj = DataFrame(
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\frame.py", line 468, in __init__
mgr = init_dict(data, index, columns, dtype=dtype)
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\internals\construction.py", line 283, in init_dict
return arrays_to_mgr(arrays, data_names, index, columns, dtype=dtype)
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\internals\construction.py", line 78, in arrays_to_mgr
index = extract_index(arrays)
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\internals\construction.py", line 397, in extract_index
raise ValueError("arrays must all be same length")
ValueError: arrays must all be same length
Simpler approach is not to save to a file and use json_normalize()
import requests
import json
import pandas as pd
data = requests.get("https://earthquake.usgs.gov/fdsnws/event/1/query?format=geojson")
json_data = data.json()
pd.json_normalize(json_data["features"])

JSON won't import

I'm creating a Discord moderator bot, and I want to create a "custom prefix per server".
My problem is that I'm trying to import json library but it doesn't work. It returns an error.
I'm using the following code (test code)
from discord.ext import commands
import discord
import json
import asyncio
with open("prefixes.json") as json_data:
prefixes = json.load(json_data)
default_prefix = "!"
def prefix(bot, message):
id = message.server.id
return prefixes.get(id, default_prefix)
bot = commands.Bot(command_prefix=prefix)
#bot.command(name="prefix", pass_context=True)
async def _prefix(ctx, new_prefix):
# Do any validations you want to do
prefixes[ctx.message.server.id] = new_prefix
with open("prefixes.json", "w") as f:
json.dump(prefixes, f)
. . .
I get this error from the code above:
===== RESTART: C:\Users\Raphaël\Desktop\Python\Moderator Bot\coreTest.py =====
Traceback (most recent call last):
File "C:\Users\Raphaël\Desktop\Python\Moderator Bot\coreTest.py", line 6, in <module>
prefixes = json.load(json_data)
File "C:\Users\Raphaël\AppData\Local\Programs\Python\Python36\lib\json\__init__.py", line 299, in load
parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, **kw)
File "C:\Users\Raphaël\AppData\Local\Programs\Python\Python36\lib\json\__init__.py", line 354, in loads
return _default_decoder.decode(s)
File "C:\Users\Raphaël\AppData\Local\Programs\Python\Python36\lib\json\decoder.py", line 339, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "C:\Users\Raphaël\AppData\Local\Programs\Python\Python36\lib\json\decoder.py", line 355, in raw_decode
obj, end = self.scan_once(s, idx)
json.decoder.JSONDecodeError: Expecting ',' delimiter: line 3 column 2 (char 30)
>>>
I found the source of the problem, I forgot a comma.
Now it's working.

json encoding issue in Python

I am attempting a custom encode, but get an error. The following code sample generates an error:
#!/usr/bin/python3
import json
class Contact:
def __init__(self, first, last):
self.first = first
self.last = last
#property
def full_name(self):
return ("{} {}".format(self.first, self.last))
class ContactEncoder(json.JSONEncoder):
def defualt(self, obj):
if isinstance(obj, Contact):
return {"is_contact": 'T'
,"first": obj.first
,"last": obj.last
,"full_name": obj.full_name}
return super().defualt(obj)
if __name__ == "__main__":
c = Contact("Jay", "Loophole")
print(json.dumps(c.__dict__))
print(json.dumps(c, cls=ContactEncoder))
The error generated is:
{"first": "Jay", "last": "Loophole"}
Traceback (most recent call last):
File "json_dump.py", line 26, in <module>
print(json.dumps(c, cls=ContactEncoder))
File "/usr/lib/python3.5/json/__init__.py", line 237, in dumps
**kw).encode(obj)
File "/usr/lib/python3.5/json/encoder.py", line 198, in encode
chunks = self.iterencode(o, _one_shot=True)
File "/usr/lib/python3.5/json/encoder.py", line 256, in iterencode
return _iterencode(o, 0)
File "/usr/lib/python3.5/json/encoder.py", line 179, in default
raise TypeError(repr(o) + " is not JSON serializable")
TypeError: <__main__.Contact object at 0x7ffb3445a400> is not JSON serializable
The default dictionary is successfully displayed, but when a custom encode is passed as a cls parameter, an error occurs.
Any suggestions for the reason for the error?
Here is your updated code after the defUAlt --> defAUlt correction:
import json
class Contact:
def __init__(self, first, last):
self.first = first
self.last = last
#property
def full_name(self):
return ("{} {}".format(self.first, self.last))
class ContactEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, Contact):
return {"is_contact": 'T'
,"first": obj.first
,"last": obj.last
,"full_name": obj.full_name}
return super().default(obj)
if __name__ == "__main__":
c = Contact("Jay", "Loophole")
print(json.dumps(c.__dict__))
print(json.dumps(c, cls=ContactEncoder))
You can check it out live on this page.

Scrapy Spider for JSON Response

I am trying to write a spider which crawls through the following JSON response:
http://gdata.youtube.com/feeds/api/standardfeeds/UK/most_popular?v=2&alt=json
How would the spider look if I would want to crawl all the titles of the videos? All my Spiders dont work.
from scrapy.spider import BaseSpider
import json
from youtube.items import YoutubeItem
class MySpider(BaseSpider):
name = "youtubecrawler"
allowed_domains = ["gdata.youtube.com"]
start_urls = ['http://www.gdata.youtube.com/feeds/api/standardfeeds/DE/most_popular?v=2&alt=json']
def parse(self, response):
items []
jsonresponse = json.loads(response)
for video in jsonresponse["feed"]["entry"]:
item = YoutubeItem()
print jsonresponse
print video["media$group"]["yt$videoid"]["$t"]
print video["media$group"]["media$description"]["$t"]
item ["title"] = video["title"]["$t"]
print video["author"][0]["name"]["$t"]
print video["category"][1]["term"]
items.append(item)
return items
I always get following error:
2014-01-05 16:55:21+0100 [youtubecrawler] ERROR: Spider error processing <GET http://gdata.youtube.com/feeds/api/standardfeeds/DE/most_popular?v=2&alt=json>
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/twisted/internet/base.py", line 1201, in mainLoop
self.runUntilCurrent()
File "/usr/local/lib/python2.7/dist-packages/twisted/internet/base.py", line 824, in runUntilCurrent
call.func(*call.args, **call.kw)
File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 382, in callback
self._startRunCallbacks(result)
File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 490, in _startRunCallbacks
self._runCallbacks()
--- <exception caught here> ---
File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 577, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/home/bxxxx/svn/ba_txxxxx/scrapy/youtube/spiders/test.py", line 15, in parse
jsonresponse = json.loads(response)
File "/usr/lib/python2.7/json/__init__.py", line 326, in loads
return _default_decoder.decode(s)
File "/usr/lib/python2.7/json/decoder.py", line 365, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
exceptions.TypeError: expected string or buffer
found two issues in your code:
start url is not accessible, I took out the www from it
changed json.loads(response) to json.loads(response.body_as_unicode())
this works well for me:
class MySpider(BaseSpider):
name = "youtubecrawler"
allowed_domains = ["gdata.youtube.com"]
start_urls = ['http://gdata.youtube.com/feeds/api/standardfeeds/DE/most_popular?v=2&alt=json']
def parse(self, response):
items = []
jsonresponse = json.loads(response.body_as_unicode())
for video in jsonresponse["feed"]["entry"]:
item = YoutubeItem()
print video["media$group"]["yt$videoid"]["$t"]
print video["media$group"]["media$description"]["$t"]
item ["title"] = video["title"]["$t"]
print video["author"][0]["name"]["$t"]
print video["category"][1]["term"]
items.append(item)
return items