Read json file using python - json

Team, i have my code working with dict in comments below. Now, i want to move that to file.json and read from there keeping the logic same. any hints how can i represent my comment section below in json and read it backin dictionary?
convert python dictionary to json file and read in python
'''
source_cidr_name = {
'location1' : ("1.1.1.1/32", [22, 443]),
'location2' : ("2.2.2.2/32", [443])}
'''
source_cidr_name = {}
with open('/Users/code/dev/whitelist.json') as jf:
source_cidr_name = json.load(jf)
file.json looks below but i don't know how to represent my ports 22 and 443
{
"source_cidr_whitelist": {"key1": {"ip_subnet": "1.1.1.1/32"}, "key2": {"ip_subnet": "2.2.2.2/32"]}}}

JSON i.e. JavaScript Object Notaion does not support including single quotes ' around strings and ( or ) arounf lists/arrays while using json.loads() to convert string representation to object (dictionary).
Using json.dumps() is okay with tuples/list, single/double quotes etc. as Python internally converts them in proper valid JSON. Mean to say it converts ( ) used to represent/surround tuples to [ ] and ' used to represent/surround strings to ".
My suggestion to get rid of this kind of problems
While dumping/writing data to JSON file use json.dumps() to create string from an existing Python objects like list/dictionary (JSON convertible).
After that if you will read the same at any point of time later, it would be easily converted to the real Python object list/dictionary etc. using json.loads() from strings or using json.load() from file like objects.
An example which helped to figure out
>>> import json
>>>
>>> d = {
... 'location1' : ("1.1.1.1/32", [22, 443]),
... 'location2' : ("2.2.2.2/32", [443])}
>>>
>>> s = json.dumps(d, indent=4)
>>> print(s)
{
"location2": [
"2.2.2.2/32",
[
443
]
],
"location1": [
"1.1.1.1/32",
[
22,
443
]
]
}
>>>
>>> new_d = json.loads(s)
>>>
>>> new_d
{u'location2': [u'2.2.2.2/32', [443]], u'location1': [u'1.1.1.1/32', [22, 443]]}
>>>
>>> new_d['location2']
[u'2.2.2.2/32', [443]]
>>>
>>> new_d['location1']
[u'1.1.1.1/32', [22, 443]]
>>>
>>> new_d['location1'][0]
u'1.1.1.1/32'
>>> new_d['location1'][1]
[22, 443]
>>>
>>> new_d['location1'][1][0]
22
>>> new_d['location1'][1][1]
443
>>>
>>> # NEW (start from string)
...
>>> s = '''{
... 'location1' : ("1.1.1.1/32", [22, 443]),
... 'location2' : ("2.2.2.2/32", [443])}'''
>>> print(s)
{
'location1' : ("1.1.1.1/32", [22, 443]),
'location2' : ("2.2.2.2/32", [443])}
>>>
1st try
>>> s = '''{'location1' : ["1.1.1.1/32", [22, 443]],'location2' : ["2.2.2.2/32", [443]]}'''
>>> s
'{\'location1\' : ["1.1.1.1/32", [22, 443]],\'location2\' : ["2.2.2.2/32", [443]]}'
>>>
>>> print(s)
{'location1' : ["1.1.1.1/32", [22, 443]],'location2' : ["2.2.2.2/32", [443]]}
>>>
>>> d = json.loads(s)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/local/Cellar/python#2/2.7.15_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/__init__.py", line 339, in loads
return _default_decoder.decode(s)
File "/usr/local/Cellar/python#2/2.7.15_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/decoder.py", line 364, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/local/Cellar/python#2/2.7.15_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/decoder.py", line 380, in raw_decode
obj, end = self.scan_once(s, idx)
ValueError: Expecting property name: line 1 column 2 (char 1)
>>>
2nd try
>>>
>>> s = '''{
... "location1" : ("1.1.1.1/32", [22, 443]),
... "location2" : ("2.2.2.2/32", [443])}'''
>>>
>>> d = json.loads(s)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/local/Cellar/python#2/2.7.15_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/__init__.py", line 339, in loads
return _default_decoder.decode(s)
File "/usr/local/Cellar/python#2/2.7.15_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/decoder.py", line 364, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/local/Cellar/python#2/2.7.15_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/decoder.py", line 382, in raw_decode
raise ValueError("No JSON object could be decoded")
ValueError: No JSON object could be decoded
>>>
>>>
Finally
>>> s = '''{
... "location1" : ["1.1.1.1/32", [22, 443]],
... "location2" : ["2.2.2.2/32", [443]]}'''
>>>
>>> d = json.loads(s)
>>> d
{u'location2': [u'2.2.2.2/32', [443]], u'location1': [u'1.1.1.1/32', [22, 443]]}
>>>
>>> type(d)
<type 'dict'>
>>>
>>> d['location2']
[u'2.2.2.2/32', [443]]
>>>
>>> d['location2'][1][0]
443
>>>

Related

Reading a JSON file using Python - JSONDecodeError Extra Data

I'm following along with https://realpython.com/python-json/. I'm using Python 3.8, on a Windows 10 machine, using IDLE.
I deviated a bit from the example.
>>> import json
>>>
>>> data1 = {
'president': {
'name': 'dumb-bell beetlebox',
'species': 'betelgusian'
}
}
>>> data2 = {
'emperor': {
'name': 'Ezekiel Wheel',
'species': 'klingon'
}
}
>>> data3 = {
'king': {
'name': 'tech shiny',
'species': 'two hundred'
}
}
>>>
>>> with open('data1_file.json', 'w') as wf:
json.dump(data1, wf)
>>> with open('data1_file.json', 'a') as af:
af.write('\n')
json.dump(data2, af)
af.write('\n')
json.dump(data3, af)
1
1
This created the json file, with the data per line.
I then tried to read it.
>>> with open('data1_file.json', 'r') as rf:
data = json.load(rf)
Traceback (most recent call last):
File "<pyshell#139>", line 2, in <module>
data4 = json.load(rf)
File "D:\Program Files (x86)\Python38-32\lib\json\__init__.py", line 293, in load
return loads(fp.read(),
File "D:\Program Files (x86)\Python38-32\lib\json\__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "D:\Program Files (x86)\Python38-32\lib\json\decoder.py", line 340, in decode
raise JSONDecodeError("Extra data", s, end)
json.decoder.JSONDecodeError: Extra data: line 2 column 1 (char 73)
On the advice from a friend, who said there may have been extraneous data in the file -
>>> print(repr(open('data1_file.json').read()))
'{"president": {"name": "dumb-bell beetlebox", "species": "betelgusian"}}\n{"emperor": {"name": "Ezekiel Wheel", "species": "klingon"}}\n{"king": {"name": "tech shiny", "species": "two hundred"}}'
Any help would be appreciated. Thank you.
The problem is json.load does not decode multiple json objects. You'll probably want to place the data in an array. Check out this link for more info

How to parse json from http response in python?

I have an API in my localhost. when I call it on browser, I will see this information:
<string xmlns="http://tempuri.org/">
{"STATUS":"202","STATUS_DT":"98/12/07","CitizenMobile":"091234567","PROFILEKEY":"1233"}
</string>
I want to use this information in my code and parse it as json. my code is:
import json
import requests
url = ""
response = requests.get(url)
print("type of response= ",type(response))
print(response.status_code)
data = response.text
parsed = json.loads(data)
print(parsed)
My output is:
type of response= <class 'requests.models.Response'>
200
Traceback (most recent call last):
File "C:/Users/MN/dev/New Project/form/WebService/TEST_MAZAHERI/Test_Stack.py", line 11, in
<module>
parsed = json.loads(data)
File "C:\Users\MN\AppData\Local\Programs\Python\Python38-32\lib\json\__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "C:\Users\MN\AppData\Local\Programs\Python\Python38-32\lib\json\decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "C:\Users\MN\AppData\Local\Programs\Python\Python38-32\lib\json\decoder.py", line 355, in
raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
I encountered this error : json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Can you help me?
just use response.json to get data in json format
import json
import requests
url = ""
response = requests.get(url)
print("type of response= ",type(response))
print(response.status_code)
data = response.json # changed here
print(data)
Try this. Works fine.
JSON is in XML, get the data, put that into xml library as string and get json from tag.
>>> data = r'<string xmlns="http://tempuri.org/"> {"STATUS":"202","STATUS_DT":"98/12/07","CitizenMobile":"091234567","PROFILEKEY":"1233"}</string>'
>>> from io import StringIO
>>> from lxml import etree
>>> root = etree.parse(StringIO(data))
>>> r = root.getroot()
>>> r.tag #also, you can iterate through particular tag, if xml has multiple tags
'{http://tempuri.org/}string'
>>> r.text #Get json text
' {"STATUS":"202","STATUS_DT":"98/12/07","CitizenMobile":"091234567","PROFILEKEY":"1233"}'
>>>
>>> import json
>>> json.loads(r.text)
{'STATUS': '202', 'STATUS_DT': '98/12/07', 'CitizenMobile': '091234567', 'PROFILEKEY': '1233'}
>>> #further operations add here.

json.loads() issue for MongoDB

I have following json file:
{'a': {'$gt': datetime.datetime(2020, 1, 1, 0, 0)}}
I converted it into string and replaces ' --> "":
>>> x
'{"a": {"$gt": datetime.datetime(2020, 1, 1, 0, 0)}}'
Getting error in loading the json:
>>> json.loads(x)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "C:\Python36\lib\json\__init__.py", line 354, in loads
return _default_decoder.decode(s)
File "C:\Python36\lib\json\decoder.py", line 339, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "C:\Python36\lib\json\decoder.py", line 357, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 15 (char 14)
The issue has got resolved. The issue arising from the fact : “datetime.datetime not JSON serialize”
from json import dumps
from datetime import date, datetime
def json_serial(obj):
"""JSON serializer for objects not serializable by default json code"""
if isinstance(obj, (datetime, date)):
return obj.isoformat()
raise TypeError ("Type %s not serializable" % type(obj))
After importing this function:
>>> x
{'a': {'$gt': datetime.datetime(2020, 1, 1, 0, 0)}}
>>> x=dumps(x,default=json_serial)
>>> x
'{"a": {"$gt": "2020-01-01T00:00:00"}}'
>>>
>>>
>>> json.loads(x)
{'a': {'$gt': '2020-01-01T00:00:00'}}
>>>

Unable to resolve TypeError: Object of type 'map' is not JSON serializable

Error while parsing map to string using json.dumps in python 3.6
x = {'id_str': '639035115457388544', 'video': False, 'photo': False, 'link': True, 'hashtags': <map object at 0x7f1762ab9320>, 'coordinates': None, 'timestamp_ms': 1441218018000, 'text': 'Police suspected hit-and-run', 'user': {'id': 628694263, 'name': 'Beth LeBlanc', 'friends_count': 235, 'verified': False, 'followers_count': 654, 'created_at': 1341631106000, 'time_zone': None, 'statuses_count': 3966, 'protected': 3966}, 'mentions': [], 'screen_name': 'THBethLeBlanc', 'reply': None, 'tweet_type': 'Tweet', 'mentionedurl': None, 'possibly_sensitive': False, 'placename': '', 'sentiments': 'Undefined'}
print(json.dumps(x))
TypeError: Object of type 'map' is not JSON serializable
I don't know how you get value for 'hashtags', but this below example will help you to solve your question a little bit. Surround your map object with list().
>>> import json
>>>
>>> some_map_value = map([],[])
>>> some_map_value
<map object at 0x7f380a75a850>
>>>
>>> x = {'hashtags': some_map_value}
>>> x
{'hashtags': <map object at 0x7f380a75a850>}
>>>
>>> json.dumps(x)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/lib/python3.7/json/__init__.py", line 231, in dumps
return _default_encoder.encode(obj)
File "/usr/lib/python3.7/json/encoder.py", line 199, in encode
chunks = self.iterencode(o, _one_shot=True)
File "/usr/lib/python3.7/json/encoder.py", line 257, in iterencode
return _iterencode(o, 0)
File "/usr/lib/python3.7/json/encoder.py", line 179, in default
raise TypeError(f'Object of type {o.__class__.__name__} '
TypeError: Object of type map is not JSON serializable
>>>
>>> list(some_map_value)
[]
>>> x = {'hashtags': list(some_map_value)} # surround your map object with list
>>> json.dumps(x)
'{"hashtags": []}'
For more information check this Getting a map() to return a list in Python 3.x
Ask Question question. If this is not you are lokking for, please put a comment to this answer.
Update: Just check your comment. Surround your map(lambda x: x['text'],doc['entities']['hashtags']) with list() like list(map(lambda x: x['text'],doc['entities']['hashtags']))
if doc['entities'].get('media'):
tweet['photo'] = True
if doc.get('extended_entities'):
tweet[doc['extended_entities']['media'][0]['type']] = True
tweet['mediaurl'] = doc['extended_entities']['media'][0]['media_url']
if doc['entities'].get('urls'):
tweet['link'] = True
tweet['hashtags'] = list(map(lambda x: x['text'],doc['entities']['hashtags']))
tweet['coordinates'] = doc['coordinates']
There is an error in your x where the hashtags key has no corresponding value. Here it is fixed:
https://repl.it/repls/SubtleLovableSystemadministrator

simplejson.scanner.JSONDecodeError: Expecting value: line 1 column 3 (char 2)

I am trying to send a http request to any url and get the response using urllib library. Following is the code that I have used :
>>> import requests
>>> r = requests.get("http://www.youtube.com/results?bad+blood")
>>> r.status_code
200
when I try to do this I get following error.
>>> r.json()
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/Library/Python/2.7/site-packages/requests/models.py", line 808, in json
return complexjson.loads(self.text, **kwargs)
File "/Library/Python/2.7/site-packages/simplejson/__init__.py", line 516, in loads
return _default_decoder.decode(s)
File "/Library/Python/2.7/site-packages/simplejson/decoder.py", line 370, in decode
obj, end = self.raw_decode(s)
File "/Library/Python/2.7/site-packages/simplejson/decoder.py", line 400, in raw_decode
return self.scan_once(s, idx=_w(s, idx).end())
simplejson.scanner.JSONDecodeError: Expecting value: line 1 column 3 (char 2)
can someone tell me whats wrong with the code.
PS: I am using python 2.7.10
The response isn't JSON, it is 'text/html; charset=utf-8'. If you want to parse it, use something like BeautifulSoup.
>>> import requests, bs4
>>> rsp = requests.get('http://www.youtube.com/results?bad+blood')
>>> rsp.headers['Content-Type']
'text/html; charset=utf-8'
>>> soup = bs4.BeautifulSoup(rsp.content, 'html.parser')
I'd recommend using the YouTube Search API instead. Log in to Google Developers Console, set up a API key following the API Key Setup instructions, then you can make the request using the YouTube Search API:
>>> from urllib import parse
>>> import requests
>>> query = parse.urlencode({'q': 'bad blood',
... 'part': 'snippet',
... 'key': 'OKdE7HRNPP_CzHiuuv8FqkaJhPI2MlO8Nns9vuM'})
>>> url = parse.urlunsplit(('https', 'www.googleapis.com',
... '/youtube/v3/search', query, None))
>>> rsp = requests.get(url, headers={'Accept': 'application/json'})
>>> rsp.raise_for_status()
>>> response = rsp.json()
>>> response.keys()
dict_keys(['pageInfo', 'nextPageToken', 'regionCode', 'etag', 'items', 'kind'])
Note that the example is using Python 3. If you want to use Python 2, then you will have to import urlencode from urllib and urlunsplit from urlparse.
That URL returns HTML, not JSON, so there's no point calling .json() on the response.