I am trying to send a http request to any url and get the response using urllib library. Following is the code that I have used :
>>> import requests
>>> r = requests.get("http://www.youtube.com/results?bad+blood")
>>> r.status_code
200
when I try to do this I get following error.
>>> r.json()
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/Library/Python/2.7/site-packages/requests/models.py", line 808, in json
return complexjson.loads(self.text, **kwargs)
File "/Library/Python/2.7/site-packages/simplejson/__init__.py", line 516, in loads
return _default_decoder.decode(s)
File "/Library/Python/2.7/site-packages/simplejson/decoder.py", line 370, in decode
obj, end = self.raw_decode(s)
File "/Library/Python/2.7/site-packages/simplejson/decoder.py", line 400, in raw_decode
return self.scan_once(s, idx=_w(s, idx).end())
simplejson.scanner.JSONDecodeError: Expecting value: line 1 column 3 (char 2)
can someone tell me whats wrong with the code.
PS: I am using python 2.7.10
The response isn't JSON, it is 'text/html; charset=utf-8'. If you want to parse it, use something like BeautifulSoup.
>>> import requests, bs4
>>> rsp = requests.get('http://www.youtube.com/results?bad+blood')
>>> rsp.headers['Content-Type']
'text/html; charset=utf-8'
>>> soup = bs4.BeautifulSoup(rsp.content, 'html.parser')
I'd recommend using the YouTube Search API instead. Log in to Google Developers Console, set up a API key following the API Key Setup instructions, then you can make the request using the YouTube Search API:
>>> from urllib import parse
>>> import requests
>>> query = parse.urlencode({'q': 'bad blood',
... 'part': 'snippet',
... 'key': 'OKdE7HRNPP_CzHiuuv8FqkaJhPI2MlO8Nns9vuM'})
>>> url = parse.urlunsplit(('https', 'www.googleapis.com',
... '/youtube/v3/search', query, None))
>>> rsp = requests.get(url, headers={'Accept': 'application/json'})
>>> rsp.raise_for_status()
>>> response = rsp.json()
>>> response.keys()
dict_keys(['pageInfo', 'nextPageToken', 'regionCode', 'etag', 'items', 'kind'])
Note that the example is using Python 3. If you want to use Python 2, then you will have to import urlencode from urllib and urlunsplit from urlparse.
That URL returns HTML, not JSON, so there's no point calling .json() on the response.
Related
I am making a script for a school project that requires that I receive a JSON file that tells me if a license plate is visible in a picture. Right now the code sends a POST with an image to an API that then gives me a JSON in return, that JSON data is sent to the file "lastResponse.json."
The code that is giving out the error
with open('lastResponse.json', 'r+') as fp:
f = json.dump(r.json(), fp, sort_keys=True, indent=4) # Where the response data is sent to the JSON
data = json.load(f) # Line that triggers the error
print(data["results"]) # Debug code
print("------------------") # Debug code
print(data) # Debug code
# This statement just checks if a license plate is visible
if data["results"]["plate"] is None:
print("No car detected!")
else:
print("Car with plate number '" + data["results"]["plate"] + "' has been detected")
The Error
Traceback (most recent call last):
File "DetectionFinished.py", line 19, in <module>
data = json.load(f)
File "/usr/lib/python3.7/json/__init__.py", line 293, in load
return loads(fp.read(),
AttributeError: 'NoneType' object has no attribute 'read'
I am not very experienced in Python so I would appreciate explanations!
It turns out, after rereading the API's documentation and using their examples I was able to fix my issues
import requests
from pprint import pprint
regions = ['gb', 'it']
with open('/path/to/car.jpg', 'rb') as fp:
response = requests.post(
'https://api.platerecognizer.com/v1/plate-reader/',
data=dict(regions=regions), # Optional
files=dict(upload=fp),
headers={'Authorization': 'Token API_TOKEN'})
pprint(response.json())
I have an API in my localhost. when I call it on browser, I will see this information:
<string xmlns="http://tempuri.org/">
{"STATUS":"202","STATUS_DT":"98/12/07","CitizenMobile":"091234567","PROFILEKEY":"1233"}
</string>
I want to use this information in my code and parse it as json. my code is:
import json
import requests
url = ""
response = requests.get(url)
print("type of response= ",type(response))
print(response.status_code)
data = response.text
parsed = json.loads(data)
print(parsed)
My output is:
type of response= <class 'requests.models.Response'>
200
Traceback (most recent call last):
File "C:/Users/MN/dev/New Project/form/WebService/TEST_MAZAHERI/Test_Stack.py", line 11, in
<module>
parsed = json.loads(data)
File "C:\Users\MN\AppData\Local\Programs\Python\Python38-32\lib\json\__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "C:\Users\MN\AppData\Local\Programs\Python\Python38-32\lib\json\decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "C:\Users\MN\AppData\Local\Programs\Python\Python38-32\lib\json\decoder.py", line 355, in
raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
I encountered this error : json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Can you help me?
just use response.json to get data in json format
import json
import requests
url = ""
response = requests.get(url)
print("type of response= ",type(response))
print(response.status_code)
data = response.json # changed here
print(data)
Try this. Works fine.
JSON is in XML, get the data, put that into xml library as string and get json from tag.
>>> data = r'<string xmlns="http://tempuri.org/"> {"STATUS":"202","STATUS_DT":"98/12/07","CitizenMobile":"091234567","PROFILEKEY":"1233"}</string>'
>>> from io import StringIO
>>> from lxml import etree
>>> root = etree.parse(StringIO(data))
>>> r = root.getroot()
>>> r.tag #also, you can iterate through particular tag, if xml has multiple tags
'{http://tempuri.org/}string'
>>> r.text #Get json text
' {"STATUS":"202","STATUS_DT":"98/12/07","CitizenMobile":"091234567","PROFILEKEY":"1233"}'
>>>
>>> import json
>>> json.loads(r.text)
{'STATUS': '202', 'STATUS_DT': '98/12/07', 'CitizenMobile': '091234567', 'PROFILEKEY': '1233'}
>>> #further operations add here.
My cmd:
python manage.py shell
from user.models import UserInfo
from user.serializers import UserInfoSerializer
import io
from rest_framework.renderers import JSONRenderer
from rest_framework.parsers import JSONParser
u=UserInfo.objects.all()[0]
s=UserInfoSerializer(u)
j=JSONRenderer().render(s.data)
o=io.BytesIO(j)
d=JSONParser().parse(o)
s1=UserInfoSerializer(data=d)
s1.is_valid()
But the issue here is that s1.is_valid() always comes out to be False and I can't save s1 as a serializer.
I'm getting this error:
>>> d
{'username': 'user001', 'password': 'pass001', 'email': 'user001#example.com', 'contact': 9876543210}
>>> s001=UserInfoSerializer(data=d)
>>> s001.is_valid()
False
>>> s001.validated_data()
Traceback (most recent call last):
File "<console>", line 1, in <module>
TypeError: 'dict' object is not callable
>>> s001.validated_data
{}
>>> s001.data
{'username': 'user001', 'password': 'pass001', 'email': 'user001#example.com', 'contact': 9876543210}
>>> s001.save()
Traceback (most recent call last):
File "<console>", line 1, in <module>
File "/home/imharjyotbagga/PycharmProjects/DRF/venv/lib/python3.7/site-packages/rest_framework/serializers.py", line 182, in save
'You cannot call `.save()` on a serializer with invalid data.'
AssertionError: You cannot call `.save()` on a serializer with invalid data.
>>>
So how can I go about this!?
You'd need to show .errors attribute to even try to diagnose the problem. My blind guess would be you messed up your password field. But you won't get any definitive answer unless you produce the code of your serializer, model, and the .errors of your serializer would also be really helpful.
There are couple of questions related to this error:
JSON object must be str, not 'bytes'
However, except obvious solution to read and decode response, I didn't learn anything special.
Here is example to the problem:
>>> import json
>>> from urllib.request import urlopen
>>> url = 'http://echo.jsontest.com/key/value/one/two'
>>> with urlopen(url) as request:
... json.load(request)
...
Traceback (most recent call last):
File "<stdin>", line 2, in <module>
File "C:\Python35\lib\json\__init__.py", line 268, in load
parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, **kw)
File "C:\Python35\lib\json\__init__.py", line 312, in loads
s.__class__.__name__))
TypeError: the JSON object must be str, not 'bytes'
So my question is why Python's JSON deserializer (that accepts file-like objects with .read() method), does not try to handle this request, as response headers hint all there is needed to know:
Content-Type: application/json; charset=ISO-8859-1
Headers hint, they do not guarantee, but that can not be a reason not to try the obvious IMHO.
Use json.loads instead of json.load
I am trying to do a simple authentication using Python3 and urllib on an API that should return account balances.
The code I have is the following:
import urllib
import urllib.request
import json
id = "nkkhuz6" # fake
secret = "s9MeR0J9yxtndLBPVA" # fake
auth_str = id + ":" + secret
def getBalances():
values = {'u' : auth_str}
data = urllib.parse.urlencode(values)
data = data.encode('utf-8') # data should be bytes
request = urllib.request.Request(url = "https://api.com", data = data)
with urllib.request.urlopen(request) as f:
print(json.loads(f.read().decode('utf-8')))
However when I run getBalances() I get the following errors:
Adriaans-MacBook-Pro:Documents adriaanjoubert$ python3 main.py
Traceback (most recent call last):
File "main.py", line 96, in <module>
getBalances()
File "main.py", line 19, in getBalances
with urllib.request.urlopen(request) as f:
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/urllib/request.py", line 161, in urlopen
return opener.open(url, data, timeout)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/urllib/request.py", line 469, in open
response = meth(req, response)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/urllib/request.py", line 579, in http_response
'http', request, response, code, msg, hdrs)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/urllib/request.py", line 507, in error
return self._call_chain(*args)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/urllib/request.py", line 441, in _call_chain
result = func(*args)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/urllib/request.py", line 587, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 405: Method Not Allowed
I am sure the URL is correct and if I append a trailing / I get the error urllib.error.HTTPError: HTTP Error 404: Not Found
When I run the following code I do get my account balances:
cmd = """curl -u """ + auth_str + """ https://api.com/"""
os.system(cmd)
What am I doing wrong? I would like to use urllib so that I can store the stuff I get back from the API in a variable.