I have a google project service account from which I got a json file. The following code works on colab:
from __future__ import print_function
import os.path
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google.oauth2 import service_account
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
import json
# If modifying these scopes, delete the file token.json.
SCOPES = ['https://www.googleapis.com/auth/spreadsheets']
# The ID and range of a sample spreadsheet.
SAMPLE_SPREADSHEET_ID = 'google_sheets_id'
SAMPLE_RANGE_NAME = 'range'
creds = None
creds = service_account.Credentials.from_service_account_info(credentials_json, scopes=SCOPES)
service = build('sheets', 'v4', credentials=creds)
values = [['planta','hshshsjh#gmail.com']]
body = {'values': values }
result = service.spreadsheets().values().append(spreadsheetId=SAMPLE_SPREADSHEET_ID,range=SAMPLE_RANGE_NAME,body=body, valueInputOption="USER_ENTERED").execute()
but not in a cloud function:
CREDENTIALS_INFO = os.environ.get("CREDENTIALS_INFO")
SCOPES = os.environ.get("SCOPES")
SPREADSHEET_ID = os.environ.get("SPREADSHEET_ID")
RANGE_NAME = os.environ.get("RANGE_NAME")
VALUE_INPUT_OPTION = os.environ.get("VALUE_INPUT_OPTION")
creds = service_account.Credentials.from_service_account_info(json.loads(CREDENTIALS_INFO), scopes=SCOPES)
service = build('sheets', 'v4', credentials=creds)
if 'planta' in request_json['queryResult']['parameters']:
values = [['planta',mail]]
body = {"values":values}
service.spreadsheets().values().append(spreadsheetId=SPREADSHEET_ID,range=RANGE_NAME,body=body, valueInputOption=VALUE_INPUT_OPTION).execute()
I get the following error:
File "/layers/google.python.pip/pip/lib/python3.10/site-packages/google/oauth2/_client.py", line 220, in jwt_grant
access_token = response_data["access_token"]
KeyError: 'access_token'
What could be happening?
Related
I've been struggling to solve a problem but could not achieve a solution.
Basically, I'm trying to read JSONs page individually, convert it to a Pandas Dataframe and then store it in a different variable for each time during a for loop.
At the end I would stack each dataframe for create a big dataframe with all the information of each page.
My code:
#Creating a list to store the URLs that will be used to connect
teste = []
for page_num in range(1, 6):
url = "pagina=" + str(page_num) + "&situacao=A&dataadmissaode=&codigoempresa=600"
teste.append(url)
#Loading libraries
import http.client
import pandas as pd
import io
import requests
import json
# auth API
conn = http.client.HTTPSConnection("Company URL")
payload = 'user=MyUser&password=MyPassword'
headers = {
'Content-Type': 'application/x-www-form-urlencoded'
}
conn.request("GET", "/login", payload, headers)
res = conn.getresponse()
data = res.read()
y = json.loads(data) #load auth and token
access_token = y.get("token", None) #just access token dynamically evey call
#second part: A for routine to execute the code for each URL generated previously in teste list.
my_dictionary = {} #empty dictionary for storing the info if possible
lista_infos = [] #empty list for storing the info if possible
for url in teste:
conn = http.client.HTTPSConnection("my_web_site.com")
payload = url
headers = {
'x-access-token': access_token,
'Content-Type': 'application/x-www-form-urlencoded'
}
conn.request("GET", "/colaborador", payload, headers)
res = conn.getresponse()
data = res.read()
#store in a dataframe and then append it to lista_infos
df_json = pd.read_json(io.StringIO(data.decode('utf-8')))
lista_infos.append(df_json)
Is there another approach to properly store the data and then create a single dataframe with the info from df_json after every call?
I've tried creating a Dataframe of information obtained with a API which does not have any documentation.
I can't create a DataFrame with the info all requests together.
I am new to Twitter stream analytics
I was unable to use the tweepy streaming as there was a change in the API version 2.0. So I am currently trying to stream it using a bearer token.
I am facing two issues:
Getting error- The content for this response was already consumed
How to send the JSON response to the Spark stream
I am streaming the JSON response by using stream=True
Any pointers/alternatives would be great!
import requests
import os
import json
bearer_token = 'your bearer token'
query = "fifa"
tweet_fields = "tweet.fields=id,created_at,text"
expansions = "expansions=author_id"
headers = {"Authorization": "Bearer {}".format(bearer_token)}
def create_url(query, tweet_fields, expansions):
url = "https://api.twitter.com/2/tweets/search/recent?query={}&{}".format(
query, tweet_fields, expansions
)
return url
def bearer_oauth(r):
"""
Method required by bearer token authentication.
"""
r.headers["Authorization"] = f"Bearer {bearer_token}"
r.headers["User-Agent"] = "v2SampledStreamPython"
return r
def connect_to_endpoint(url):
response = requests.request("GET", url, auth=bearer_oauth, stream=True)
#print(response.status_code)
for response_line in response.iter_lines():
if response_line:
json_response = json.loads(response_line)
t=json.dumps(json_response, indent=4, sort_keys=True)
if response.status_code != 200:
raise Exception(
"Request returned an error: {} {}".format(
response.status_code, response.text
)
)
def main():
url = create_url(query, tweet_fields, expansions)
timeout = 0
while True:
connect_to_endpoint(url)
timeout += 1
if __name__ == "__main__":
main()
I have a lambda function which filters out running ec2 instances and stores them in an array, then it is sent as a payload to the slack channel. The problem is when I run the function it will not send the list but the print command will list it out on output.
import json
import boto3
import logging
import os
import urllib3
import urllib.parse
#from urlparse import parse_qs
from boto3 import resource
ec2 = boto3.resource('ec2')
region = 'us-east-1'
def lambda_handler(event, context):
ec2 = boto3.resource('ec2', region)
url = "https://hooks.slack.com/services/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"
text = "hi"
filters = [
{
'Name': 'instance-state-name',
'Values': ['running']
}
]
instances = ec2.instances.filter(Filters = filters)
RunningInstances = []
for instance in instances:
RunningInstances.append(instance.id)
instanceList = json.dumps(RunningInstances)
#List out running ec2 instances
print(instanceList)
http = urllib3.PoolManager()
r = http.request("POST",
"https://hooks.slack.com/services/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",
body = json.dumps(instanceList),
headers = {"Content-Type":"application/json"})
```
I need to scrape some data for the Dart / Flutter application and I need to log in to access the page.
How do I submit changes to the form data and click on the login button? I tried as follows:
var loginPage = await http.get('https://mypage.com/login');
var document = parse(loginPage.body);
var username = document.querySelector('#username') as InputElement;
var password = document.querySelector('#password') as InputElement;
username.value = 'USERNAME';
password.value = 'PASSWORD';
var submit = document.querySelector('.btn-submit') as ButtonElement;
submit.click();
But I have the following error:
Error: 'InputElement' isn't a type.
Error: 'ButtonElement' isn't a type.
I also tried the following:
InputElement username = document.querySelector('#username');
But a get the error A value of type 'Element' can't be assigned to a variable of type 'InputElement'
I need to make this scrape in the flutter application to avoid passing the password using API.
How can I log in to the page to get the data?
InputElement and ButtonElement are part of the dart:html package, therefore, we shouldn't forget to import it in our code:
import 'dart:html';
import 'package:http/http.dart' as http;
void submitForm() async {
var loginPage = await http.get('https://mypage.com/login');
var document = parse(loginPage.body);
var username = document.querySelector('#username') as InputElement;
var password = document.querySelector('#password') as InputElement;
username.value = 'USERNAME';
password.value = 'PASSWORD';
var submit = document.querySelector('.btn-submit') as ButtonElement;
submit.click();
}
By doing so, the compiler will recognize InputElement and ButtonElement as correct types.
Another, more elegant, way to obtain the same result would be:
import 'dart:html';
import 'package:http/http.dart' as http;
void submitForm() async {
var loginPage = await http.get('https://mypage.com/login');
Document document = parse(loginPage.body);
(document.querySelector('#username') as InputElement).value = 'USERNAME';
(document.querySelector('#password') as InputElement).value = 'PASSWORD';
(document.querySelector('.btn-submit') as ButtonElement).click();
}
If we'd like to use the same functionality in a Flutter project, we might need the help of the universal_html package:
Add this to your package's pubspec.yaml file:
dependencies:
universal_html: ^1.2.3
Run flutter pub get
Import the package and use the code:
import 'package:http/http.dart' as http;
import 'package:universal_html/html.dart';
void submitForm() async {
var loginPage = await http.get('https://mypage.com/login');
Document document = parse(loginPage.body);
(document.querySelector('#username') as InputElement).value = 'USERNAME';
(document.querySelector('#password') as InputElement).value = 'PASSWORD';
(document.querySelector('.btn-submit') as ButtonElement).click();
}
I am quite stuck in trying to subscribe to changes on a google drive folder. My python3 code is as follows:
SCOPES = 'https://www.googleapis.com/auth/drive.readonly'
store = file.Storage('storage.json')
credentials = store.get()
if not credentials or credentials.invalid:
flow = client.flow_from_clientsecrets('client_id.json', SCOPES)
credentials = tools.run_flow(flow, store)
# This starts the authorization process
DRIVE = discovery.build('drive', 'v3', http=credentials.authorize(Http()))
try:
with open('saved_start_page_token.json') as json_data:
d = json.load(json_data)
try:
saved_start_page_token = d["startPageToken"]
except KeyError:
saved_start_page_token = d["newStartPageToken"]
print("Using saved token: %s" % saved_start_page_token)
except FileNotFoundError:
response = DRIVE.changes().getStartPageToken().execute()
with open("saved_start_page_token.json", "w") as token:
json.dump(response, token)
saved_start_page_token = response.get('startPageToken')
print('Start token: %s' % saved_start_page_token)
body = dict()
body["kind"] = "api#channel"
body["id"] = str(uuid.uuid4()) # TODO: do I have to do something with this channel id?
print(body["id"])
body["resourceId"] = 'web_hook'
body["resourceUri"] = 'https://meg-wm-it-change.appspot.com/notifications/'
json_body = json.dumps(body)
print(json_body)
request = DRIVE.changes().watch(pageToken = saved_start_page_token, body=json_body)
response = request.execute()
return response.body
Except this throws the error
googleapiclient.errors.HttpError: <HttpError 400 when requesting https://www.googleapis.com/drive/v3/changes/watch?pageToken=163958&alt=json returned "entity.resource">
Which I can't quite decipher. I'm sure my issue will be not understanding the documentation, (ie, I don't understand whether the params go versus the body of this request, and can't find any code samples) but any help would be appreciated!
I am going to post the answer I found to my own question in the event that anyone else wanders here:
# If modifying these scopes, delete the file token.pickle.
SCOPES = ['https://www.googleapis.com/auth/drive']
def auth():
creds = None
# The file token.pickle stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
if os.path.exists('token.pickle'):
with open('token.pickle', 'rb') as token:
creds = pickle.load(token)
# If there are no (valid) credentials available, let the user log in.
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(
'credentials.json', SCOPES)
creds = flow.run_local_server()
# Save the credentials for the next run
with open('token.pickle', 'wb') as token:
pickle.dump(creds, token)
return creds
def subscribe_changes(service):
channel_id = str(uuid.uuid4())
body = {
"id": channel_id,
"type": "web_hook",
"address": COOL_REGISTERED_DOMAIN
}
response = service.changes().watch(body=body, pageToken = get_page_token(service)).execute()
ts = response['expiration']
print(dateparser.parse(ts))
print(response)
return channel_id
def main():
creds = auth()
service = build('drive', 'v3', credentials=creds)
subscribe_changes(service)