I've made some basic code to explain my question:
produceUTF8.py (replies 'ít wórked!' with unicode characters) - you run this first
# -*- coding: utf-8 -*-
import os
from sys import argv
from flask import Flask, request, Response, jsonify
import json
app = Flask(__name__)
app.config['JSON_AS_ASCII'] = False # contribution from Erdem
#app.route('/reply', methods=['POST'])
def reply():
"""Fetch a reply
"""
print("DEBUG entered")
params = request.json
print("DEBUG entered2")
if not params:
return jsonify({
'status': 'error',
'error': 'Request must be of the application/json type!',
})
reply = "ít wórked!"
# Send the response.
return jsonify({
'status': 'ok',
'reply': reply,
})
if __name__ == '__main__':
app.run(host='0.0.0.0', debug=True)
consumeUTF8.py (posts the message 'óíá' to get answer from producer)
# -*- coding: utf-8 -*-
import requests
HEADERS = {'Content-Type': 'application/json; charset=utf-8',}
DATA = '{"message": "óíá"}'
my_request = requests.post('http://localhost:5000/reply', headers=HEADERS, data=DATA)
response = my_request.json()['reply']
In my producer I am getting "Bad Request (400)" and in the consumer "json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)."
It seems to be a problem in params = request.json as seen in the debug prints. What is the recommended approach here?
Thanks!
You can fix the way you make the request by encoding the data object :
my_request = requests.post('http://localhost:5000/reply',
headers=HEADERS,
data=DATA.encode('utf-8'))
#>>> ít wórked with óíá!
If you add a try/except statement in the app, it returns :
try:
params = request.json
except Exception as e:
params = None
print(e)
400 Bad Request: Failed to decode JSON object: 'utf-8' codec can't decode byte 0xf3 in position 14: invalid continuation byte
You could use this pattern to assign a default value for param
I have not tried the code but may be setting
app.config['JSON_AS_ASCII'] = False
may help.
Related
Okay, so this is a loaded question but and I'm sure theres an easy method to use here, but I'm stuck.
Long story short, I am tasked with creating a function in python (to be run an AWS lambda) which can perform acceptance tests on a series of URL's using python-requests. These requests will be used to assert the HTTP response codes and a custom HTTP header identifying if an haproxy backend is correct.
The URL's themselves will be maintained in a yaml document which will be converted to a dict in python and passed to a for loop which will use python requests to HTTP GET the response code and header of the URL.
The issue I am having is getting a single body object to return the results of multiple for loops.
I have tried to find similar use cases but cannot
import requests
import json
import yaml
def acc_tests():
with open("test.yaml", 'r') as stream:
testurls = yaml.safe_load(stream)
results = {}
# endpoint/path 1
for url in testurls["health endpoints"]:
r = requests.get(url, params="none")
stat = r.status_code
result = json.dumps(print(url, stat))
results = json.dumps(result)
# endpoint path with headers
for url in testurls["xtvapi"]:
headers = {'H': 'xtvapi.cloudtv.comcast.net'}
r = requests.get(url, headers=headers, params="none")
stat = r.status_code
head = r.headers["X-FINITY-TANGO-BACKEND"]
result = json.dumps((url, stat, head))
results = json.dumps(result)
return {
'statusCode': 200,
'body': json.dumps(results)
}
acc_tests()
YAML file:
health endpoints:
- https://xfinityapi-tango-production-aws-us-east-1-active.r53.aae.comcast.net/tango-health/
- https://xfinityapi-tango-production-aws-us-east-1-active.r53.aae.comcast.net/
- https://xfinityapi-tango-production-aws-us-east-2-active.r53.aae.comcast.net/tango-health/
- https://xfinityapi-tango-production-aws-us-east-2-active.r53.aae.comcast.net/
- https://xfinityapi-tango-production-aws-us-west-2-active.r53.aae.comcast.net/tango-health/
- https://xfinityapi-tango-production-aws-us-west-2-active.r53.aae.comcast.net/
xtvapi:
- https://xfinityapi-tango-production-aws-us-east-1-active.r53.aae.comcast.net/
- https://xfinityapi-tango-production-aws-us-east-2-active.r53.aae.comcast.net/
- https://xfinityapi-tango-production-aws-us-west-2-active.r53.aae.comcast.net/
What I think is happening is that both for loops are running one after another, but the value of results is empty, but I'm not sure what to do in order to update/append the results dict with the results of each loop.
Thanks folks. I ended up solving this by creating a dict with immutable keys for each test type and then using append to add the results to a nested list within the dict.
Here is the "working" code as it is in the AWS Lambda function:
from botocore.vendored import requests
import json
import yaml
def acc_tests(event, context):
with open("test.yaml", 'r') as stream:
testurls = yaml.safe_load(stream)
results = {'tango-health': [], 'xtvapi': []}
# Tango Health
for url in testurls["health endpoints"]:
r = requests.get(url, params="none")
result = url, r.status_code
assert r.status_code == 200
results["tango-health"].append(result)
# xtvapi default/cloudtv
for url in testurls["xtvapi"]:
headers = {'H': 'xtvapi.cloudtv.comcast.net'}
r = requests.get(url, headers=headers, params="none")
result = url, r.status_code, r.headers["X-FINITY-TANGO-BACKEND"]
assert r.status_code == 200
assert r.headers["X-FINITY-TANGO-BACKEND"] == "tango-big"
results["xtvapi"].append(result)
resbody = json.dumps(results)
return {
'statusCode': 200,
'body': resbody
}
I am trying to access the values from an API json item called DataClasses that looks like this.
"DataClasses":["Dates of birth","Email addresses","IP addresses","Passwords","Usernames","Website activity"]
With this code:
import requests
import json
with open('dataclasstest.txt', 'r') as f:
emails = [line.strip() for line in f]
print("Email" + ":" + "Data Classes")
def main():
for name in emails:
url = 'https://haveibeenpwned.com/api/v2/breachedaccount/' + name
headers = {
'User-Agent': 'dataclass.py'
'Accept: application/vnd.haveibeenpwned.v2+json'
}
r = requests.get(url, headers=headers)
content = r.json()
data = json.loads(content)
if r.status_code == 200:
for item in data:
print(unicode(item['DataClasses']))
time.sleep(2)
elif r.status_code == 404:
print("No Data")
time.sleep(2)
else:
time.sleep(2)
if __name__ = "__main__":
main()
Can anyone help me get on the right track? I've been looking online for weeks and still can't figure it out. I've found similar answers on here but when I try them it still doesn't work. Thanks in advance!
Here is how I did it. I will post only the relevant lines of code:
r = requests.get(url, headers=headers)
print(json.loads(r.content)[0].get('DataClasses'))
After reading the docs for requests the code above can be rewritten more simpler.
r = requests.get(url, headers=headers)
print(r.json()[0].get('DataClasses'))
Because we know that the response represents JSON data, requests has a built-in JSON decoder. r.json() returns a list of objects, or a list of dict. We then extract the first element with r.json()[0]. The .get method is exactly the same as calling r.json()[0]['DataClasses']. You can read more about it here.
I'm trying to set-up a TCP echo client and server that can exchange messages using the JSON format.
I took the code from the documentation and modified it as follows:
Edit: include fix and have both server and client send JSON style messages.
import asyncio
# https://docs.python.org/3/library/asyncio-stream.html
import json
async def handle_echo(reader, writer):
data = await reader.read(100)
message = json.loads(data.decode())
addr = writer.get_extra_info('peername')
print("Received %r from %r" % (message, addr))
print("Send: %r" % json.dumps(message)) # message
json_mess_en = json.dumps(message).encode()
writer.write(json_mess_en)
#writer.write(json_mess) # not wokring
#writer.write(json.dumps(json_mess)) # not working
# Yielding from drain() gives the opportunity for the loop to schedule the write operation
# and flush the buffer. It should especially be used when a possibly large amount of data
# is written to the transport, and the coroutine does not yield-from between calls to write().
#await writer.drain()
#print("Close the client socket")
writer.close()
loop = asyncio.get_event_loop()
coro = asyncio.start_server(handle_echo, '0.0.0.0', 9090, loop=loop)
server = loop.run_until_complete(coro)
# Serve requests until Ctrl+C is pressed
print('Serving on {}'.format(server.sockets[0].getsockname()))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
# Close the server
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
and the client code:
import asyncio
import json
async def tcp_echo_client(message, loop):
reader, writer = await asyncio.open_connection('0.0.0.0', 9090,
loop=loop)
print('Send: %r' % message)
writer.write(json.dumps(message).encode())
data = await reader.read(100)
data_json = json.loads(data.decode())
print('Received: %r' % data_json)
print(data_json['welcome'])
print('Close the socket')
writer.close()
message = {'welcome': 'Hello World!'}
loop = asyncio.get_event_loop()
loop.run_until_complete(tcp_echo_client(message, loop))
loop.close()
Error
TypeError: data argument must be a bytes-like object, not 'str'
Should I use another function than writer.write to encode for JSON? Or any suggestions?
Found the solution, replace:
writer.write(json.dumps(json_mess))
for
# encode as 'UTF8'
json_mess_en = json.dumps(json_mess).encode()
writer.write(json_mess_en)
I am writing a Flask application in which I have a service that generates a JWT and passes this onto another service using requests.post(), after decoding it to 'UTF-8'.
While sending the JWT, I can see that the type is 'str'. However, on performing a json.loads() on the other service, I get an error that says
TypeError: the JSON object must be str, not 'bytes'
Here is my code:
Service 1:
#app.route('/')
def index():
token = jwt.encode({'message': 'Hello'}, app.config['SECRET_KEY'])
# After this statement I am able to verify the type is str and not bytes
token = token.decode('UTF-8')
headers = {'content-type': 'application/json'}
url = 'someUrl'
data = {"token": token}
data = json.dumps(data)
requests.post(url, data=data, headers=headers)
return 'Success'
Service 2:
#app.route('/', methods=['POST'])
def index():
data = json.loads(request.data)
return 'Success'
Why do I get this error even though the type was converted to string ?
EDIT: I was able to successfully retrieve the token by passing it through header. But I would still like to know what caused this error.
You could post it as JSON instead of data, and let the underlying library take care of it for you.
Service 1
#app.route('/')
def index():
token = jwt.encode({'message': 'Hello'}, app.config['SECRET_KEY']).decode('UTF-8')
url = 'someUrl'
data = {"token": token}
requests.post(url, json=data)
return 'Success'
Service 2
data = request.get_json()
while running this script i am getting the following error...
ERROR: getTest failed. No JSON object could be decoded
My code is:
import json
import urllib
class test:
def getTest(self):
try:
url = 'http://www.exapmle.com/id/123456/'
json_ = urllib.urlopen(url).read()
self.id = json.loads(json_)
print self.id
except Exception, e:
print "ERROR: getTest failed. %s" % e
if __name__ == "__main__":
ti = test()
ti.getTest()
while opening this url "http://www.exapmle.com/id/123456/" in browser i am getting json format data, but why it is throwing an error?
Print the json_ before calling json.loads(). If there is in an error in your url, urllib does not necessarily throw an exception, it might just retrieve some error page (like with the provided url), that is not valid json.