Creating new dataframes for every loop - json

I've been struggling to solve a problem but could not achieve a solution.
Basically, I'm trying to read JSONs page individually, convert it to a Pandas Dataframe and then store it in a different variable for each time during a for loop.
At the end I would stack each dataframe for create a big dataframe with all the information of each page.
My code:
#Creating a list to store the URLs that will be used to connect
teste = []
for page_num in range(1, 6):
url = "pagina=" + str(page_num) + "&situacao=A&dataadmissaode=&codigoempresa=600"
teste.append(url)
#Loading libraries
import http.client
import pandas as pd
import io
import requests
import json
# auth API
conn = http.client.HTTPSConnection("Company URL")
payload = 'user=MyUser&password=MyPassword'
headers = {
'Content-Type': 'application/x-www-form-urlencoded'
}
conn.request("GET", "/login", payload, headers)
res = conn.getresponse()
data = res.read()
y = json.loads(data) #load auth and token
access_token = y.get("token", None) #just access token dynamically evey call
#second part: A for routine to execute the code for each URL generated previously in teste list.
my_dictionary = {} #empty dictionary for storing the info if possible
lista_infos = [] #empty list for storing the info if possible
for url in teste:
conn = http.client.HTTPSConnection("my_web_site.com")
payload = url
headers = {
'x-access-token': access_token,
'Content-Type': 'application/x-www-form-urlencoded'
}
conn.request("GET", "/colaborador", payload, headers)
res = conn.getresponse()
data = res.read()
#store in a dataframe and then append it to lista_infos
df_json = pd.read_json(io.StringIO(data.decode('utf-8')))
lista_infos.append(df_json)
Is there another approach to properly store the data and then create a single dataframe with the info from df_json after every call?
I've tried creating a Dataframe of information obtained with a API which does not have any documentation.
I can't create a DataFrame with the info all requests together.

Related

How can I pass variable(m_otp) here in my payload

import requests
import random
m_otp=str(random.randint(100000,999999))
print(m_otp)
url = "https://ziper.io/api/send.php?instance_id=My_instance_id&access_token=My_Acess_token&type=json&number=919XXXXXXXXX"
payload = "{\r\n \"text\": \"Hello there \"\r\n }"
headers = {}
response = requests.request("POST", url, headers=headers, data=payload)
print(response.text)
I tried solutions from github but there is the thing I'm getting it.
Do you think of something like:
import requests
import random
import json
m_otp=str(random.randint(100000,999999))
print(m_otp)
url = "https://ziper.io/api/send.php?instance_id=My_instance_id&access_token=My_Acess_token&type=json&number=919XXXXXXXXX"
payload = {"text": "Hello there", "m_otp":m_otp}
payload_str = json.dumps(payload)
headers = {}
response = requests.request("POST", url, headers=headers, data=payload_str)
print(response.text)
?
If not, feel free to tell me the issue and I'll edit my answer.
Also, shouldn't m_otp get passed as a int ?

Unable to stream data from twitter using Pyspark

I am new to Twitter stream analytics
I was unable to use the tweepy streaming as there was a change in the API version 2.0. So I am currently trying to stream it using a bearer token.
I am facing two issues:
Getting error- The content for this response was already consumed
How to send the JSON response to the Spark stream
I am streaming the JSON response by using stream=True
Any pointers/alternatives would be great!
import requests
import os
import json
bearer_token = 'your bearer token'
query = "fifa"
tweet_fields = "tweet.fields=id,created_at,text"
expansions = "expansions=author_id"
headers = {"Authorization": "Bearer {}".format(bearer_token)}
def create_url(query, tweet_fields, expansions):
url = "https://api.twitter.com/2/tweets/search/recent?query={}&{}".format(
query, tweet_fields, expansions
)
return url
def bearer_oauth(r):
"""
Method required by bearer token authentication.
"""
r.headers["Authorization"] = f"Bearer {bearer_token}"
r.headers["User-Agent"] = "v2SampledStreamPython"
return r
def connect_to_endpoint(url):
response = requests.request("GET", url, auth=bearer_oauth, stream=True)
#print(response.status_code)
for response_line in response.iter_lines():
if response_line:
json_response = json.loads(response_line)
t=json.dumps(json_response, indent=4, sort_keys=True)
if response.status_code != 200:
raise Exception(
"Request returned an error: {} {}".format(
response.status_code, response.text
)
)
def main():
url = create_url(query, tweet_fields, expansions)
timeout = 0
while True:
connect_to_endpoint(url)
timeout += 1
if __name__ == "__main__":
main()

How do I decode a DRF response object's content to a Python String?

I have the following route in Django Rest Framework:
from rest_framework.viewsets import ModelViewSet
from rest_framework.renderers import JSONRenderer
from rest_framework.response import Response
class MainViewset(ModelViewSet):
renderer_classes = [JSONRenderer]
authentication_classes = []
permission_classes = []
def alive(self, request):
return Response("API is Alive", 200)
I have a Django test that calls this API route, expecting the JSON string:
def test_base_route(self):
c = Client()
response = c.get('/budget/alive')
self.assertTrue(response.status_code == 200)
self.assertEqual(response.content.decode("UTF-8"), "API is Alive")
However, I get the following error:
def test_base_route(self):
c = Client()
response = c.get('/budget/alive')
self.assertTrue(response.status_code == 200)
> self.assertEqual(response.content.decode("UTF-8"), "API is Alive")
E AssertionError: '"API is Alive"' != 'API is Alive'
E - "API is Alive"
E ? - -
E + API is Alive
I find this strange since I decoded the string. I know it's a simple thing to trim off quotation marks, but what is the right way to serialize a single string as a response and get it back in the content of a response in DRF when sending JSON?
You can use .data for this case:
self.assertEqual(response.data, "API is Alive")

How to get a better formatted JSON Response in DjangoRestFramework UI

I have an APIVIEW in DRF which I have written some views in it to get some Response, I would like to get a better formatted JSONResponse.
Views.py
class Pay(APIView):
def get(self, request):
url = "https://api.paystack.co/transaction/verify/262762380"
payload = {}
files = {}
headers = {
'Authorization': 'Bearer SECRET_KEY',
'Content-Type': 'application/json'
}
response = requests.request("GET", url, headers=headers, data= payload, files=files)
return Response(response)
This is a pictorial representation of the bad formatted JSONResponse I am getting which I would like to improve. Thanks
So in the case you don't have a Model but rather some data structure you get in some other way (as in a response from a third party API) just return a Response with this data structure as argument.
In this case you are effectively acting as a kind of proxy between the 3rd party API and your client, if that is what you intend (be sure not to leak private data!)
Example:
class Pay(APIView):
def get(self, request):
url = "https://api.paystack.co/transaction/verify/262762380"
payload = {}
files = {}
headers = {
'Authorization': 'Bearer SECRET_KEY',
'Content-Type': 'application/json'
}
# no need for 'data' and 'files' arguments with method GET
response = requests.get(url, headers=headers)
# do some validation, at least check for response.status_code
if response.status_code == 200:
data = response.json()
return Response(data)
There are many more possibilities when using the requests library.

How to send a file object from Flask API to Angular Frontend without storing it on the file system

I am trying send a file object of type BytesIO from flask API to angular frontend.
I am using json.dumps(). File object shouldn't be stored on file system. I am using memory for storing file object. Using io.BytesIO().
return json.dumps({'UserId': username, 'file': file_object}), 201
Typerror: <_io.BytesIO object> is not json searializable
You can use send_file with a file like object:
import io
from flask import send_file
#app.route("/send_file")
def send_file():
file = io.BytesIO()
file.write(b"Hello, World!")
file.seek(0)
return send_file(file, attachment_filename=f"example.txt", as_attachment=True)
And then I expect you'll do something with it in Javascript:
fetch('https://example.com/send_file')
.then(res => res.blob())
.then(blob => {
// Do something
});
http://blog.luisrei.com/articles/flaskrest.html
Please check this link and refer "RESPONSES" subtitle.
from flask import Response
#app.route('/hello', methods = ['GET'])
def api_hello():
data = {
'hello' : 'world',
'number' : 3
}
js = json.dumps(data)
resp = Response(js, status=200, mimetype='application/json')
resp.headers['Link'] = 'http://luisrei.com'
return resp