error loading json using topsy - json

When i load single record json is created just fine when i try to load multiple records i get this error. Sorry i am new to python http://tny.cz/ce1baaba
Traceback (most recent call last):
File "TweetGraber.py", line 26, in <module>
get_tweets_by_query(topic)
File "TweetGraber.py", line 15, in get_tweets_by_query
json_tree = json.loads(source)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/__init__.py", line 338, in loads
return _default_decoder.decode(s)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/decoder.py", line 368, in decode
raise ValueError(errmsg("Extra data", s, end, len(s)))
ValueError: Extra data: line 2 column 1 - line 11 column 1 (char 2380 - 46974)
Here is my code
def get_tweets_by_query(query, count=10):
"""A function that gets the tweets by a query."""
Tweets=[]
queryEncoded=urllib.quote(query)
api_key = "xxxxx"
source=urllib.urlopen("http://api.topsy.com/v2/content/bulktweets.json?q=%s&type=tweet&offset=0&perpage=%s&window=realtime&apikey=%s" % (queryEncoded, count, api_key)).read()
json_tree = json.loads(source)
pprint(json_tree)
topic = raw_input("Please enter a topic: ")
get_tweets_by_query(topic)

Thanks Timusan I was able to correct my json The problem with the original it was missing the root element "[" which showed we are expecting array and there "," was missing after end of each object. So here is fixed code.
So here is the code
def get_tweets_by_query(query, count=10):
"""A function that gets the tweets by a query."""
Tweets=[]
queryEncoded=urllib.quote(query)
api_key = "xx"
source=urllib.urlopen("http://api.topsy.com/v2/content/bulktweets.json?q=%s&type=tweet&offset=0&perpage=%s&window=realtime&apikey=%s" % (queryEncoded, count, api_key)).read()
source="["+source+"]"
source=source.replace("}\n{","},{")
json_tree = json.loads(source)
pprint(json_tree)

Related

Python requests:- simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

I am trying to get some order details using the python requests library. The API is working fine with the postman, but when I try to run it with python the following error occurs.
simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
I am using the following python code to call the API in the screenshots:
import requests
url = "https://xxxxxxxxxxxxxx/purchases"
header = {"Content-Type":"application/json", "Authorization":'token {}'.format('eyXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX')}
data = {"from_date":"2021-06-15T07:19:32", "to_date":"2021-09-16 00:01:09"}
response = requests.get(url, params=data, headers=header)
print(response.json())
The result I am getting is:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/lib/python3/dist-packages/requests/models.py", line 897, in json
return complexjson.loads(self.text, **kwargs)
File "/usr/lib/python3/dist-packages/simplejson/__init__.py", line 518, in loads
return _default_decoder.decode(s)
File "/usr/lib/python3/dist-packages/simplejson/decoder.py", line 370, in decode
obj, end = self.raw_decode(s)
File "/usr/lib/python3/dist-packages/simplejson/decoder.py", line 400, in raw_decode
return self.scan_once(s, idx=_w(s, idx).end())
simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
The issue was I need to add a text "Bearer " in front of the token passed in the Autorization parameter of header section as the following because the token is to be passed as a Bearer Token:
header = {"Content-Type":"application/json", "Authorization":'Bearer eyXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'}

Json loads fails on mqtt message

I'm trying to parse the messages sent on an MQTT topic as JSONs, but when i try to call json.loads() on the decoded string, the code goes into exception:
File "/usr/lib/python3.7/json/decoder.py", line 340, in decode
raise JSONDecodeError("Extra data", s, end)
json.decoder.JSONDecodeError: Extra data: line 1 column 153 (char 152)
The code is the following:
def on_message_mqtt(self, client, userdata, message):
m_decode = message.payload.decode('utf-8')
logger.info(m_decode)
m_in = json.loads(m_decode)
logger.info(m_in)
While the payload of the messages has the following format:
{"Timestamp":"2021-05-24-13:27:13.450","AI":[1204,7,0,4,18,2,1176,802,11,0,381,2496,0,0,810,1282],"DI":[false,false,false,false,false,false,false,false]}
I tried to put the text into a string and loads() is able to parse it, so I bet that the problem is on the initial string decoding.
Here you can see a print of the decoded string, and then the error on the call to json.loads():
2021-05-24 16:31:47 - INFO - {"Timestamp":"2021-05-24-14:31:44.790","AI":[912,9,0,2,16,2,886,605,11,0,321,1924,1,963,620,9],"DI":[false,false,false,false,false,false,false,false]}
Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/lib/python3.7/threading.py", line 917, in _bootstrap_inner
self.run()
File "/usr/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "router.py", line 136, in run
self.mqtt.loop_forever()
File "/opt/control/env/lib/python3.7/site-packages/paho/mqtt/client.py", line 1779, in loop_forever
rc = self.loop(timeout, max_packets)
File "/opt/control/env/lib/python3.7/site-packages/paho/mqtt/client.py", line 1181, in loop
rc = self.loop_read(max_packets)
File "/opt/control/env/lib/python3.7/site-packages/paho/mqtt/client.py", line 1572, in loop_read
rc = self._packet_read()
File "/opt/control/env/lib/python3.7/site-packages/paho/mqtt/client.py", line 2310, in _packet_read
rc = self._packet_handle()
File "/opt/control/env/lib/python3.7/site-packages/paho/mqtt/client.py", line 2940, in _packet_handle
return self._handle_pubrel()
File "/opt/control/env/lib/python3.7/site-packages/paho/mqtt/client.py", line 3246, in _handle_pubrel
self._handle_on_message(message)
File "/opt/control/env/lib/python3.7/site-packages/paho/mqtt/client.py", line 3444, in _handle_on_message
self.on_message(self, self._userdata, message)
File "router.py", line 87, in on_message_mqtt
logger.info(json.loads(m_decode), exc_info=True)
File "/usr/lib/python3.7/json/__init__.py", line 348, in loads
return _default_decoder.decode(s)
File "/usr/lib/python3.7/json/decoder.py", line 340, in decode
raise JSONDecodeError("Extra data", s, end)
json.decoder.JSONDecodeError: Extra data: line 1 column 151 (char 150)
Judging from the error, character 152 is the square bracket closing the arroy of the value for the key of "DI" ] which would imply incorrect parsing using the payload.decode function
Can you check if this works
def on_message_mqtt(self, client, userdata, message):
m_in = json.loads(message.decode("utf-8"))
logger.info(m_in)

How to skip text at the beginning of a json file in python

I have a json object file and at the beginning of this file a header of text, date, time, IP address.
I've tried f.readlines()[5:] to no avail.
I've tried next(f)
I wish to skip 5 or 6 lines of text and go directly into the json data.
Here is an example.
import jsonlines
import json
data_file = input("Enter a file to parse: ")
with jsonlines.open(data_file) as file:
for obj in file:
try:
jsonparse = json.loads(obj)
except Exception as e:
pass
print(obj)
Error:
jsonlines.jsonlines.InvalidLineError: line contains invalid json: Expecting value: line 1 column 1 (char 0) (line 1)
Top of json file:
Start: 07/02/2019 14:59:40.686
Connected To:
192.168.11.203
Here is the full long error:
Enter a file to parse: Play.raw
Traceback (most recent call last):
File "C:\Users\sdickey\AppData\Local\Programs\Python\Python37\lib\site-packages\jsonlines\jsonlines.py", line 159, in read
value = self._loads(line)
File "C:\Users\sdickey\AppData\Local\Programs\Python\Python37\lib\json\__init__.py", line 348, in loads
return _default_decoder.decode(s)
File "C:\Users\sdickey\AppData\Local\Programs\Python\Python37\lib\json\decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "C:\Users\sdickey\AppData\Local\Programs\Python\Python37\lib\json\decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:/Users/sdickey/PycharmProjects/Python Testing/Actall Data/testing.py", line 6, in <module>
for obj in file:
File "C:\Users\sdickey\AppData\Local\Programs\Python\Python37\lib\site-packages\jsonlines\jsonlines.py", line 204, in iter
skip_empty=skip_empty)
File "C:\Users\sdickey\AppData\Local\Programs\Python\Python37\lib\site-packages\jsonlines\jsonlines.py", line 164, in read
six.raise_from(exc, orig_exc)
File "<string>", line 3, in raise_from
jsonlines.jsonlines.InvalidLineError: line contains invalid json: Expecting value: line 1 column 1 (char 0) (line 1)
Process finished with exit code 1
It is always preferable not to read the whole file in memory but one line at a time.
Assuming your input file contains:
first line
second line
third line
fourth line
fifth line
{ "k1": "val1", "k2": "val2" }
{ "k3": "val3", "k4": "val4" }
if you just want to skip 5 lines, you could do it brutally as:
import json
with open("test.txt") as f:
for _ in range(5):
next(f)
for line in f:
obj = json.loads(line)
print(obj)
or using enumerate:
import json
with open("test.txt") as f:
for i, line in enumerate(f):
if i<5:
continue
obj = json.loads(line)
print(obj)
or use itertools' dropwhile:
import itertools as it
import json
with open("test.txt") as f:
for i, line in it.dropwhile(lambda i: i[0]<5, enumerate(f)):
obj = json.loads(line)
print(obj)
I think you are trying to convert to json line by line. You need to join all lines by \n igoring the first 5 lines and then load it:
import json
with open("test.txt") as f:
json_obj = "\n".join(f.readlines()[5:])
jsonparse = json.loads(json_obj)
print(jsonparse)

App randomly crashes with a JSON error

I've posted first the error message with full traceback and the code as well.
The code that is below is within a loop, that will run multiple times changing the variables within the session.get() link for the API data call.
From time to time I will get an error that the json.decoder.JSONDecodeError is expecting value... (full error below)
I can't figure out why.
Traceback (most recent call last):
File "C:\Users\CarlosMiguel\Desktop\experiments\data_collector\data_collector.py", line 100, in <module>
get_data = session.get(api_data_link+symbol+"?limit=1000&period="+i).json()
File "C:\Users\CarlosMiguel\Anaconda3\lib\site-packages\requests\models.py", line 892, in json
return complexjson.loads(self.text, **kwargs)
File "C:\Users\CarlosMiguel\Anaconda3\lib\json\__init__.py", line 354, in loads
return _default_decoder.decode(s)
File "C:\Users\CarlosMiguel\Anaconda3\lib\json\decoder.py", line 339, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "C:\Users\CarlosMiguel\Anaconda3\lib\json\decoder.py", line 357, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Currently It's written like this (everything that needs to be is imported):
try:
get_data = session.get(api_data_link+symbol+"?limit=1000&period="+i).json()
for candle in get_data:
#Do things with the requested data....
except:
print("JSONError")
print('Could not update data for')
print(symbol + ' ' + i)
Could it be that the API just doesn't return anything on some random call?
Because it seems to be completely random when it crashes with that error.
If it could be the API just not returning anything, how could I write it to call it again for a few times, before continuing on with the rest of the app?

urllib2.URLError: <urlopen error [Errno 8]

import urllib2
import urllib
import json
url = "http://ajax/googleapis.com/ajax/services/search/web?v=1.0&"
query = raw_input ("What do you want to search for ? >> ")
query = urllib.urlencode({'q': query})
response = urllib2.urlopen (url + query).read()
data = json.loads (response)
results = data ['responseData'] ['results']
for result in results:
title = result['title']
url = result['url']
print (title + ';' + url)
ERROR
/System/Library/Frameworks/Python.framework/Versions/2.6/bin/python2.6 /Users/dragonleo/PycharmProjects/untitled2/googleapi
What do you want to search for ? >> apple
Traceback (most recent call last):
File "/Users/dragonleo/PycharmProjects/untitled2/googleapi", line 8, in
response = urllib2.urlopen (url + query).read()
File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/urllib2.py", line 391, in open
response = self._open(req, data)
File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/urllib2.py", line 409, in _open
'_open', req)
File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/urllib2.py", line 369, in _call_chain
result = func(*args)
File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/urllib2.py", line 1181, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/urllib2.py", line 1156, in do_open
raise URLError(err)
urllib2.URLError:
Appreciate if expert can explain why I am getting the error
Two problems stand out immediately:
There are multiple typos in the code above. Specifically, there are no spaces between brackets and parens. Also, the URL should be ajax.googleapis.com.
The Google Web Search API is no longer available. You should migrate to the Google Custom Search API