"errorMessage": "Decimal('1') is not JSON serializable" - json

import boto3
import json
import os
s3 = boto3.client('s3')
ddb = boto3.resource('dynamodb')
table = ddb.Table('test_table')
def lambda_handler(event, context):
response = table.scan()
body = json.dumps(response['Items'])
response = s3.put_object(Bucket='s3-testing',
Key = 's3-testing.json' ,
Body=body,
ContentType='application/json')
lambda function
Hi guys I am trying to run this code but I keep receiving this TypeError: Decimal('1') is not JSON serializable. When I play around and import Decimal, it gives me this "errorMessage": "Unable to import module 'lambda_function'"
I am assuming I have to add simplejson but do not know how to go around it.
execution error

Related

BlockingIOError: [Errno 11] write could not complete without blocking

Please help, I have developed this scraper API and it works with internal commands in Pythionanywhere, but when I try to access it using my use account URL, it gives the error: [errno 11] and I have searched for solutions, I co[enter image description here](https://i.stack.imgur.com/cs9T6.png)uldn't find any.
I was expecting a JSON output as it did on the internal server.
#from requests_html import HTMLSession
import json
import requests
class Scraper():
def scrapedata(self, tag):
url = "https://www.etenders.gov.za/Home/TenderOpportunities/?status=1"
headers = {'user-agent': 'Mozilla/5.0'}
response = requests.get(url, headers=headers)
data = response.json()
file_urs = []
for e in data:
item = {
'province': (f"{e['province']}"),
'id' : (f"{e['tender_No']}"),
}
print(file_urs)
file_urs.append(item)
return file_urs
quotes = Scraper()
quotes.scrapedata('cat')
from flask import Flask
import json
from scrape import Scraper
app = Flask(name)
quotes = Scraper()
#app.route('/', methods =['GET', 'POST'])
async def read_item(cat):
json_dumps = json.dumps.scrapedata(cat)
return json_dumps
#return quotes.scrapedata(cat).jon()
from flask import Flask
import json
from scrape import Scraper
app = Flask(name)
quotes = Scraper()
#app.route('/', methods =['GET', 'POST'])
async def read_item(cat):
json_dumps = json.dumps.scrapedata(cat)
return json_dumps
#return quotes.scrapedata(cat).jon()
I was expecting a JSON output as it did on the internal server.

How to create PyArrow Table from IOBytes stream of a JSON file from s3 bucket

import io
from pyarrow.json import read_json
import boto3
session = boto3.Session(aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
s3 = session.resource('s3', endpoint_url=SCALITY_ENDPOINT, verify=r'C:\path\to\verification\certificate.crt')
bucket = s3.Bucket('my_bucket')
object = bucket.Object('my_object.json')
file_stream = io.BytesIO()
object.download_fileobj(file_stream)
table = read_json(file_stream.getvalue())
But I get the following error:
TypeError: Cannot convert bytes to pyarrow.lib.NativeFile
Is there a way to create a PyArrow table directly from the stream?

Python: request json gets error - If using all scalar values, you must pass an index

When attempting to request a json file from API, i have an error after get the first result.
Does anyone have any idea why is requested an index from a list ?
Best regards :)
import requests
import json
import pandas as pd
import time
import datetime
### OCs List ids
OCs = ['1003473-1116-SE21','1003473-1128-AG21','1031866-12-CC21','1057440-3184-AG21','1070620-1832-CM21', '1070620-2219-SE21', '1070620-2499-CM21']
for i in OCs:
link ="http://api.mercadopublico.cl/servicios/v1/publico/ordenesdecompra.json?codigo="+ str(i) +"&ticket=F8537A18-6766-4DEF-9E59-426B4FEE2844"
response = requests.get(link, [])
data = response.json()
df = pd.DataFrame.from_dict(data)
### remove unnecessary columns
df.drop(df.columns[[0,1,2]],axis=1, inplace=True)
### flat json to pandas dataframe
df_detail = pd.json_normalize(df['Listado'])
ValueError: If using all scalar values, you must pass an index

The server detects too many requests and sends error response (and then your script throws an error). Solution is to wait for correct response, for example:
import requests
import json
import pandas as pd
import time
import datetime
### OCs List ids
OCs = [
"1003473-1116-SE21",
"1003473-1128-AG21",
"1031866-12-CC21",
"1057440-3184-AG21",
"1070620-1832-CM21",
"1070620-2219-SE21",
"1070620-2499-CM21",
]
for i in OCs:
link = (
"http://api.mercadopublico.cl/servicios/v1/publico/ordenesdecompra.json?codigo="
+ str(i)
+ "&ticket=F8537A18-6766-4DEF-9E59-426B4FEE2844"
)
while True: # <--- repeat until we get correct response
print(link)
response = requests.get(link, [])
data = response.json()
if "Listado" in data:
break
time.sleep(3) # <--- wait 3 seconds and try again
df = pd.DataFrame.from_dict(data)
### remove unnecessary columns
df.drop(df.columns[[0, 1, 2]], axis=1, inplace=True)
### flat json to pandas dataframe
df_detail = pd.json_normalize(df["Listado"])
# ...

I am having trouble converting my nested json into a dataframe. I am getting the json from an API and want it in a dataframe

This code is from Sportradar API. The API outputs the data as JSON or XML; below is my attempt at taking the JSON and making it into a dataframe.
import numpy as np
import pandas as pd
import http.client
import json
from pandas.io.json import json_normalize
#API Call including my key
conn = http.client.HTTPSConnection("api.sportradar.us")
conn.request("GET", "/nfl/official/trial/v5/en/players/0acdcd3b-5442-4311-a139-ae7c506faf88/profile.json?api_key=99s3ewmn5rrdrd9r3v5wrfgd")
#conn.request("GET", "/nfl/official/trial/v5/en/games/b7aeb58f-7987-4202-bc41-3ad9a5b83fa4/pbp.json?api_key=99s3ewmn5rrdrd9r3v5wrfgd")
#conn.request("GET", "/nfl/official/trial/v5/en/teams/0d855753-ea21-4953-89f9-0e20aff9eb73/full_roster.json?api_key=99s3ewmn5rrdrd9r3v5wrfgd")
#conn.request("GET", "/nfl/official/trial/v5/en/games/030d37cf-b896-4f10-b16e-2a5120fef6cf/pbp.json?api_key=99s3ewmn5rrdrd9r3v5wrfgd")
res = conn.getresponse()
data = res.read()
data_dec = data.decode("utf-8")
json_data = json.loads(data_dec)
flat_data = json_normalize(json_data)
print(json_data)
df = pd.DataFrame.from_records(flat_data)
df2 = pd.DataFrame.from_dict(json_data, orient='index')
df2.reset_index(level=0, inplace=True)
#The closest thing to a dataframe I can get
df.head()

Why not make use of a Python Wrapper that is publicly available and maintained.
See link.

"TypeError: Object of type 'bytes' is not JSON serializable" live streaming data

I am trying to live stream data into Power Bi from python. However I am encountering the error
TypeError: Object of type 'bytes' is not JSON serializable
I have put my code below, please indicate what I am doing wrong as I don't quite understand what the issue is.
import pandas as pd
from datetime import datetime
from datetime import timedelta
import requests
import json
import time
import random
# function for data_generation
def data_generation():
surr_id = random.randint(1, 3)
speed = random.randint(20, 200)
date = datetime.today().strftime("%Y-%m-%d")
time = datetime.now().isoformat()
return [surr_id, speed, date, time]
if __name__ == '__main__':
REST_API_URL = 'api_url'
while True:
data_raw = []
for j in range(1):
row = data_generation()
data_raw.append(row)
print("Raw data - ", data_raw)
# set the header record
HEADER = ["surr_id", "speed", "date", "time"]
data_df = pd.DataFrame(data_raw, columns=HEADER)
data_json = bytes(data_df.to_json(orient='records'), encoding='utf-8')
print("JSON dataset", data_json)
# Post the data on the Power BI API
try:
req = requests.post(REST_API_URL, data=json.dumps(
data_json), headers=HEADER, timeout=5)
print("Data posted in Power BI API")
except requests.exceptions.ConnectionError as e:
req = "No response"
print(req)
time.sleep(3)

Solved, Just changed req = requests.post(REST_API_URL, data=json.dumps(data_json), headers=HEADER, timeout=5) to req = requests.post(url=REST_API_URL, data=data_json)