I'm sending following REST request to the Read The Docs API:
GET /api/v3/projects/<my_project>/redirects/
I receive JSON in the response body, but I'm unable to parse the JSON into something useful.
I'm using a Python3 program:
import requests
import json
URL = 'https://readthedocs.com/api/v3/projects/<my_project>/redirects/'
TOKEN = "<my_access_token>"
HEADERS = {'Authorization': f'token {TOKEN}'}
response = requests.get(URL, headers=HEADERS, stream=True)
print(f'Response is {response.json()}')
This program returns something that looks like JSON.
If I run it using the following command line:
python3 rtd_get_redirects.py | python3 -m json.tool
I receive the following error:
Expecting value: line 1 column 1 (char 0)
The entire response is large, so I'll just paste this snippet from the beginning:
{'count': 521, 'next': 'https://readthedocs.com/api/v3/projects/signalfx-product-docs/redirects/?limit=10&offset=10', 'previous': None, 'results': [{'_links': {'_self': 'https://readthedocs.com/api/v3/projects/signalfx-product-docs/redirects/6612/', 'project': 'https://readthedocs.com/api/v3/projects/signalfx-product-docs/'}, 'created': '2022-07-20T18:08:52.133088Z',
Related
I haven't found the docs for that use case. How can I get the request body, ensure it's a valid JSON (any valid JSON, including numbers, string, booleans, and nulls, not only objects and arrays) and get the actual JSON.
Using Pydantic forces the JSON to have a specific structure.
You can find nearly everything inside the Request object
You are able to get request body with request.json(), which will give you the parsed JSON as dictionary.
from fastapi import Request, FastAPI
#app.post("/dummypath")
async def get_body(request: Request):
return await request.json()
If you want access the body as string, you can use request.body()
The accepted answer is valid as well, but FastAPI provides a built-in way to do that - check the Singular values in body section in docs.
A parameter with the default Body gets all the payload that doesn't match passed Pydantic-typed parameters (the whole payload in our case) and converts it to the dict. In case of invalid JSON, a standard validation error would be produced.
from fastapi import Body, FastAPI
app = FastAPI()
#app.post('/test')
async def update_item(
payload: dict = Body(...)
):
return payload
UPD: Note on ... (Ellipsis) - it allows marking a value as required. Read more in the Required with Ellipsis docs section
If you are confident that the incoming data is "a valid JSON", you can create a simple type annotation structure to receive the arbitrary JSON data.
from fastapi import FastAPI
from typing import Any, Dict, AnyStr, List, Union
app = FastAPI()
JSONObject = Dict[AnyStr, Any]
JSONArray = List[Any]
JSONStructure = Union[JSONArray, JSONObject]
#app.post("/")
async def root(arbitrary_json: JSONStructure = None):
return {"received_data": arbitrary_json}
Examples
1. JSON object
curl -X POST "http://0.0.0.0:6022/" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"test_key\":\"test_val\"}"
Response:
{
"received_data": {
"test_key": "test_val"
}
}
2. JSON array
curl -X POST "http://0.0.0.0:6022/" -H "accept: application/json" -H "Content-Type: application/json" -d "[\"foo\",\"bar\"]"
Response:
{
"received_data": [
"foo",
"bar"
]
}
If you are not sure about the content type of the incoming data, better to parse the request body.
It can be done as,
from fastapi import FastAPI, Request
app = FastAPI()
#app.post("/")
async def root(request: Request):
return {"received_request_body": await request.body()}
The advantage of this method is that the body will contain any kind of data, JSON, form-data, multipart-form-data, etc.
from fastapi import Request
async def synonyms__select(request: Request):
return await request.json()
will return a JSON object.
This is an example to print the content of a Request, it will print the json body (if it is json parsable) otherwise just print the raw bytes of the body.
async def print_request(request):
print(f'request header : {dict(request.headers.items())}' )
print(f'request query params : {dict(request.query_params.items())}')
try :
print(f'request json : {await request.json()}')
except Exception as err:
# could not parse json
print(f'request body : {await request.body()}')
#app.post("/printREQUEST")
async def create_file(request: Request):
try:
await print_request(request)
return {"status": "OK"}
except Exception as err:
logging.error(f'could not print REQUEST: {err}')
return {"status": "ERR"}
FastAPI has a JSON encoder.
There are some cases where you might need to convert a data type (like
a Pydantic model) to something compatible with JSON
from fastapi import FastAPI
from fastapi.encoders import jsonable_encoder
from pydantic import BaseModel
import simplejson as json
class SubmitGeneral(BaseModel):
controllerIPaddress: str
readerIPaddress: str
ntpServer: str
#app.post("/submitGeneral")
async def submitGeneral(data: SubmitGeneral):
data = jsonable_encoder(data)
#data = json.loads(data.json()) # same as above line
print(f"data = {json.dumps(data)}")
# you have to access the properties with brackets, not by dot notation
query = f"update LocalPLC set ControllerIpAddress = '{data['controllerIPaddress']}', ReaderIPAddress = '{data['readerIPaddress']}'"
return {"status": "OK"}
For those of you using BaseModel and want to have a JSON field, you can import Json from pydantic
from fastapi import FastAPI
from pydantic import BaseModel, Json, Field
app = FastAPI()
class MockEndpoint(BaseModel):
endpoint: str = Field(description="API endpoint to mock")
response: Json = Field(description="Example response of the endpoint")
#app.get("/")
async def root():
return {"message": "Hello World"}
#app.post("/mock")
async def mock_request(mock_endpoint: MockEndpoint):
return mock_endpoint
I use the code below in python on a rpi to do json posts
import requests
payload = '{"msg_type":"observation_payload","msg":
{"serial":"FZtank","observations":{"8":{"d":0,"m":"JSON upload
test","t":"1543540780"}}}}'
r = requests.post('https://my.website/api/observations', data =
{'post_wrapper':payload})
print r.text
it produces
{"msg": {"remote_pi": "FZtank", "stored_ids": ["8"]}, "msg_type": "receipt"}
I figured it wouldn't be too hard to do the same thing in micropython on a pycom device
import urequests as requests
payload = '{"msg_type":"observation_payload","msg":
{"serial":"FZtank","observations":{"8":{"d":0,"m":"JSON upload
test","t":"1543540780"}}}}'
r = requests.post('https://my.website/api/observations', data =
{"post_wrapper":payload})
print(r.text)
r.close()
but it gives
Traceback (most recent call last):
File "<stdin>", line 12, in <module>
File "/flash/lib/urequests.py", line 115, in post
File "/flash/lib/urequests.py", line 100, in request
File "/flash/lib/urequests.py", line 79, in request
TypeError: object with buffer protocol required
The TypeError msg means it wants a string instead of a dictionary for the data
so I tried
import urequests as requests
payload = '{"post_wrapper":{"msg_type":"observation_payload","msg":
{"serial":"FZtank","observations":{"8":{"d":0,"m":"JSON upload
test","t":"1543540780"}}}}}'
r = requests.post('https://my.website/api/observations', data =payload)
print(r.text)
r.close()
but that gives
{"msg": {"errorText": "Malformed POST - unable to decode JSON"}, "msg_type":
"error"}
So now I'm confused. Is the problem that the website expects a dictionary & micropython can only send a string?
.
Some further detail:
I'm using https://github.com/micropython/micropython-lib/blob/master/urequests/urequests.py but with line 52 changed from ai = usocket.getaddrinfo(host, port, 0, usocket.SOCK_STREAM) to ai = usocket.getaddrinfo(host, port) because it used to throw an error msg about 4 arguments Vs 2. If I run
import urequests as requests
payload = '{"post_wrapper":{"msg_type":"observation_payload","msg":
{"serial":"FZtank","observations":{"8":{"d":0,"m":"JSON upload
test","t":"1543540780"}}}}}'
r = requests.post('https://my.website/api/observations', data=payload)
or
import urequests as requests
payload = {"post_wrapper":{"msg_type":"observation_payload","msg":
{"serial":"FZtank","observations":{"8":{"d":0,"m":"JSON upload
test","t":"1543540780"}}}}}
r = requests.post('https://my.website/api/observations', json=payload)
or
import urequests as requests
payload = {"msg_type":"observation_payload","msg":
{"serial":"FZtank","observations":{"8":{"d":0,"m":"JSON upload
test","t":"1543540780"}}}}
r = requests.post('https://my.website/api/observations', json=
{"post_wrapper":payload})
I get the same result
{"msg": {"errorText": "Malformed POST - unable to decode JSON"}, "msg_type":
"error"}
I was stuck with the same issue here while trying to send push notifications through pushed.co.
I was using data=payload on urequests and getting this TypeError: object with buffer protocol required error. All worked well when I changed to json=payload following #stijn suggestion on his answer.
Steps I did:
on micropython REPL install urequests
import upip
upip.install('micropython-urequests')
for my example the code is
import urequests as requests
payload = {
"app_key": "your_app_key",
"app_secret": "your_app_secret",
"target_type": "your_target_type",
"content": "Hello from micropython",
}
r = requests.post("https://api.pushed.co/1/push", json=payload)
Probably any other url will do fine too.
I was able to make posts request on micropython dumping data into a json:
import json
import urequests
data = {"mykey": 'myvalue'}
urequests.post(url, data=json.dumps(data))
I've made some basic code to explain my question:
produceUTF8.py (replies 'ít wórked!' with unicode characters) - you run this first
# -*- coding: utf-8 -*-
import os
from sys import argv
from flask import Flask, request, Response, jsonify
import json
app = Flask(__name__)
app.config['JSON_AS_ASCII'] = False # contribution from Erdem
#app.route('/reply', methods=['POST'])
def reply():
"""Fetch a reply
"""
print("DEBUG entered")
params = request.json
print("DEBUG entered2")
if not params:
return jsonify({
'status': 'error',
'error': 'Request must be of the application/json type!',
})
reply = "ít wórked!"
# Send the response.
return jsonify({
'status': 'ok',
'reply': reply,
})
if __name__ == '__main__':
app.run(host='0.0.0.0', debug=True)
consumeUTF8.py (posts the message 'óíá' to get answer from producer)
# -*- coding: utf-8 -*-
import requests
HEADERS = {'Content-Type': 'application/json; charset=utf-8',}
DATA = '{"message": "óíá"}'
my_request = requests.post('http://localhost:5000/reply', headers=HEADERS, data=DATA)
response = my_request.json()['reply']
In my producer I am getting "Bad Request (400)" and in the consumer "json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)."
It seems to be a problem in params = request.json as seen in the debug prints. What is the recommended approach here?
Thanks!
You can fix the way you make the request by encoding the data object :
my_request = requests.post('http://localhost:5000/reply',
headers=HEADERS,
data=DATA.encode('utf-8'))
#>>> ít wórked with óíá!
If you add a try/except statement in the app, it returns :
try:
params = request.json
except Exception as e:
params = None
print(e)
400 Bad Request: Failed to decode JSON object: 'utf-8' codec can't decode byte 0xf3 in position 14: invalid continuation byte
You could use this pattern to assign a default value for param
I have not tried the code but may be setting
app.config['JSON_AS_ASCII'] = False
may help.
A few days ago I did a similar question here: How to get JSON data in an Odoo controller?
But now, I need to create a controller which receives only JSON data. So, I am doing the request from a Python console, this way:
import requests
import json
url = 'http://localhost:8069/odoo/test'
headers = {'Content-Type': 'application/json'}
data = {
'name': 'Jane',
'email': 'jane.doe#gmail.com',
}
data_json = json.dumps(data)
r = requests.post(url=url, data=data_json, headers=headers)
I have created a controller which listens to http://localhost:8069/odoo/test, this way:
import openerp.http as http
from openerp.http import Response
import logging
_logger = logging.getLogger(__name__)
class WebFormController(http.Controller):
#http.route('/odoo/test', type='json',
auth='public', methods=['POST'], website=True)
def index(self, **args):
_logger.info('CONNECTION SUCCESSFUL')
_logger.info(args)
name = args.get('name', False)
email = args.get('email', False)
_logger.info(name)
_logger.info(email)
if not name:
Response.status = '400 Bad Request'
return '{"response": "OK"}'
The problem is that I am receiving an empty JSON in the controller. I can read CONNECTION SUCCESFUL in the log, with no error, but when I show args, I get {}, and obviously due to that, False when writing name and email.
If I pass the data as a Python dictionary or as a string, I get the following error:
Invalid JSON data: 'name=Jane&email=jane.doe%40gmail.com' or
Invalid JSON data: "{'name': 'Jane', 'email': 'jane.doe#gmail.com'}" respectively.
If I modify type='json' and I write type='http' instead, I get the following error:
Function declared as capable of handling request of type 'http' but called with a request of type 'json'.
I have read that may be this could be solved if the request is sent using the parameter json instead of data, this way:
r = requests.post(url=url, json=data_json, headers=headers)
Unfortunately, the server which is going to make the request has an old operating system which cannot update the python-requests package, so I cannot use json parameter since it did not exist at the version installed in that server.
Please, can anyone help me? I need get JSON data in the controller, not a string neither Python dictionaries.
You have just forgotten to put your data inside the params keywords:
Use this correct syntax :
data = {"params": dict(key="value")}
data = {
"params": {
"name":"prakashsharma",
"email":"prakashsharmacs24#gmail.com",
"phone":"+917859884833"
}
}
Please don't forget to use json.dumps(data) and 'Content-Type': 'application/json' while requesting a resource in json format.
I am damn sure your issue will be solved after using this one my friend... cheers :)!!
You can use below format for a POST request
{
"params" : {
"name" : "Order/1/18",
"session_id" : 1,
"customer_count" : 2,
"partner_id" : 9,
"lines": [
{
"product_id": 37,
"qty" : 2,
"price_unit" : 2,
"discount" : 10
}
],
"pos_reference" : 2,
"note" : "This is a test note"
}
}
Content type must be application/json
How odoo route will handle request ?
Route will help creating a POS order in odoo [POST]
#http.route(['/api/v1/resources/<string:api_key>/pos_order'],
auth="public",
website=False,
type="json",
csrf=False,
methods = ['POST'])
def create_update_pos_order(self, api_key=None, **kwargs):
print(kwargs.get('name')) -> Order/1/18
God Bless Forvas::
But for more clearity:
if you want to test through cURL:
curl -i -X POST -H "Content-Type: application/json" -d '{"params": {"name":"prakashsharma","email":"prakashsharmacs24#gmail.com","phone":"+917859884833"}}' 'http://localhost:8069/web/yourlistoner/'
if you want to test through python request:
import requests
headers = {
'Content-Type': 'application/json',
}
data = '{"params": {"name":"prakashsharma","email":"prakashsharmacs24#gmail.com","phone":"+917859884833"}}'
requests.post('http://localhost:8069/web/yourlistoner/', headers=headers, data=data)
the function in odoo will be something like
from odoo import http
import json
class YourClass(http.Controller):
#http.route('/web/yourlistoner/', type='json', auth="none", methods=['POST'],cors="*", csrf=False)
def listoner(self, **kw):
print http.request.params
print "lllllllllllllllllllll"
return json.dumps({"result":"Success"})
I am a newbie to python and am trying to create a script to login to crucible and use the token to pass to other services.
1) I am able to make a xml request and get a response but as soon as I pass the headers to my conn.request it says HTTP Error 415, unsupported Media Type.
I have done quiet a bit of research on this topic and found out that the rest API might not be supporting the json reques, but Crucible says that there API supports json so seems to be some other issue,
2) when trying to pass the args generated using feauth the auth token is not getting used , for now I have appended it to url and it works.
Please help me with the same , below is my script
import httplib
import urllib
import json
from xml.etree.ElementTree import XML
import xml.dom.minidom
conn = httplib.HTTPSConnection("fisheye")
args=urllib.urlencode({'userName':'UNAME', 'password':'PWD'})
headers={'content-type':'application/json', 'accept':'application/json'}
#headers={'Authorization' : 'Basic %s' % base64.b64encode("username:password")}
r1 = conn.request("post", "/rest-service/auth-v1/login", args)
#status = r1[u'headers']['status']
#conn.connect()
r2 = conn.getresponse()
print r1,r2.status,r2.reason,r2
r3=r2.read()
print(r3)
r4=str(r3)
print r4
data = XML(r4).find("token").text
print data
# data1=urllib.quote_plus(data, safe=":")
# print data1
args=urllib.urlencode({'FEAUTH':data}).replace("%3A", ":")
print "args is", args
#args={}
req = conn.request("get","/rest-service/reviews-v1")
r3 = conn.getresponse()
status = r3.status
print "the url is"#, r3.getheader('Location')
url=r3.getheader('location', '')
print url
url1=r3.msg#.dict['location']
print url1
#print req.url
#print req.get_method()
print dir(req) # list lots of other stuff in Request
print "after sending open review request"
print r3
print req,r3.status,r3.reason,r3
r4=r3.read()
print(r4)
r5=str(r4)
print r5
# json_ob=json.loads(r3.read())
# print json_ob
I was able to resolve the issue by
1) removing the Content-Type from the headers and changed the accept to Accept(sentence cased).
2) The login request was a get request and hence it supports data transfer by URL append, it is only for post request that we can pass an argument.
In the header of the request, try to specify the media type:
headers = { 'Content-Type' : 'application/json' }
req = urllib2.Request(url, headers=headers)