Why "Unexpected Error" on UrlFetchApp GAS - google-apps-script

My code has an issue when run by trigger but running manually is ok.
i have the following function
var url = "https://api.facebook.com/method/links.getStats?urls="+links+"&format=json";
var getParams = {
"method": "get",
"validateHttpsCertificates":false,
"accept": "application/json",
"muteHttpExceptions": true
};
var stats = UrlFetchApp.fetch(url,getParams);
Utilities.sleep(3000);
var cont = JSON.parse(stats.getContentText());
Running by trigger it logs "Unexpected Error"
according to my research i found that is an usual issue, i read that the rules to run manually are diferent than by trigger (even the ip are diferent)
"Apps Script uses two different UrlFetchApp pipelines: one for when the code is run by a user and one for when the code is run by a trigger. The trigger pipeline has some slightly different rules, which is why you are occasionally seeing these errors."
Well i think this is a serious issue and if google do it on purpose to regulate Fetch calls, i think is an offense for users and basically should not exist trigger service.
Any idea for the Solution?

I am found a posible temporal solution:
Util.fetchJSON = function (url) {
var maxTries = 3;
var tries = 0;
var res, body;
do {
tries++;
if (tries > 1) {
Utilities.sleep(1000);
Logger.log('GET %s (try #%s)', url, tries);
} else {
Logger.log('GET %s', url);
}
res = UrlFetchApp.fetch(url);
body = res.getContentText();
} while (!body && (tries < maxTries));
if (!body) {
throw new Error('Unable to fetch JSON after ' + tries + ' attempts: ' + url);
}
return JSON.parse(body);
};
But i dont know how to apply it to my original function: (help)
(this is a part of my original function)
function soyEspiritual() {
do {
try {
var pipe = "http://pipes.yahoo.com/pipes/pipe.run?_id=888ed60ff6a5ee79c05ec6963f6d3efe&_render=json";
var pipedata = UrlFetchApp.fetch(pipe,{method:"get"});
Utilities.sleep(3000);
var object = JSON.parse(pipedata.getContentText());

Related

Google App Scripts / TwitterLib getting error Exception when trying to post a tweet

Trying to implement TwitterLib to send tweets from Google Sheets. I am using Google App Scripts, and the following code -
var sheet = SpreadsheetApp.getActive().getSheetByName('View');
var startRowNumber = 1;
var endRowNumber = sheet.getLastRow();
function sendTweets() {
var twitterKeys = {
TWITTER_CONSUMER_KEY: "xxxxxxxxxxxxxxxxxxx",
TWITTER_CONSUMER_SECRET: "xxxxxxxxxxxxxxxxxxx",
TWITTER_ACCESS_TOKEN: "xxxxxxxxxxxxxxxxxxx",
TWITTER_ACCESS_SECRET: "xxxxxxxxxxxxxxxxxxx"
}
var props = PropertiesService.getScriptProperties();
props.setProperties(twitterKeys);
var params = new Array(0);
var service = new Twitterlib.OAuth(props);
var quote;
var identifier;
for (var currentRowNumber = startRowNumber; currentRowNumber <= endRowNumber; currentRowNumber++) {
var row = sheet.getRange(currentRowNumber + ":" + currentRowNumber).getValues();
// check that the second column (Date) is equal to today
if (isToday(row[0][1])) {
quote = row[0][0];
identifier = currentRowNumber - 1;
if (!service.hasAccess()) {
console.log("Authentication Failed");
} else {
console.log("Authentication Successful");
var status = quote + "\n\n" + "#Quotes #Motivation";
try {
var response = service.sendTweet(status, params);
console.log(response);
} catch (e) { console.log(e) }
}
break;
}
}
}
function isToday(date) {
var today = new Date();
var dateFromRow = new Date(date);
return dateFromRow.getDate() == today.getDate() &&
dateFromRow.getMonth() == today.getMonth() &&
dateFromRow.getFullYear() == today.getFullYear()
}
I have signed up for Twitter DEV and have my API Key and Secret (CONSUMER_KEY and CONSUMER_SECRET above) and Access Token and Access Secret as well. I have turned on OAuth 1.0a, Read and write and Direct Message selected, and a Callback URI of (https://script.google.com/macros/s/YOUR_SCRIPT_ID/usercallback) - Replacing YOUR_SCRIPT_ID with the actual one I have.
I am seeing an Authentication Successful message in my Console, but seeing this error when running inside Apps Scripts IDE:
Send tweet failure. Error was: {"name":"Exception"}
Nothing more. I am not sure what else to check to see what I am doing wrong. Any help or resources to read over would greatly be appreciated! Thank you so much!
JJ

Writing a request JSON for fetchURL in Google Scripts: Script runs twice?

sorry in advance for the long question. I am trying to create a Google Sheet that tells me how many hours each of my contractors has logged on Clockify each the month. (Full code at the bottom)
In short my problem is creating a JSON file for the UrlFetchApp.fetch() request to the Clockify API using input from the google sheet.
I want the JSON to look like this:
var newJSON = {
"dateRangeStart": "2022-01-01T00:00:00.000",
"dateRangeEnd": "2022-01-31T23:59:59.000",
"summaryFilter": {
"groups": ["USER"],
"sortColumn": "GROUP"
}
}
var payload = JSON.stringify (newJSON);
And when I use this code, it works perfectly. However, the start and end dates are variables that I compute in the google sheet, as I need these dates to change each month. I wrote a function that gives me the correct outputs ("2022-01-01T00:00:00.000", "2022-01-31T23:59:59.000"), but when I reference the cells in google sheets, I get a 400 error saying that the API was not able to parse the JSON.
Function in Script:
function GetHours(userName, startDate, endDate) {
var newJSON = {
"dateRangeStart": startDate,
"dateRangeEnd": endDate,
"summaryFilter": {
"groups": ["USER"],
"sortColumn": "GROUP"
}
}
var payload = JSON.stringify (newJSON);
...}
Calling the function in sheets:
=GetHours(C3,$D$45,$D$46)
Output error message:
Exception: Request failed for https://reports.api.clockify.me returned code 400. Truncated server response: {"code":400,"message":"generateSummaryReport.arg1.dateRangeEnd: Field dateRangeEnd is required, generateSummaryReport.arg1.dateRangeStart: Field da... (use muteHttpExceptions option to examine full response)
A weird thing is happening when I use Logger.log(payload), which may be the root of the problem. It appears that the code runs twice, and the first time the payload JSON is correct, but the second it is incorrect.
First time:
{"dateRangeStart":"2022-01-01T00:00:00.000","dateRangeEnd":"2022-01-31T23:59:59.000","summaryFilter":{"groups":["USER"],"sortColumn":"GROUP"}}
Second time:
{"summaryFilter":{"groups":["USER"],"sortColumn":"GROUP"}}
I have tried a bunch of solutions, but really it boils down to referencing the Google sheet in the JSON. When I copy and paste the output of my date calculation into the JSON, it works. When I create a variable in Scripts with the date calculation output, it works. When I return startDate, it gives me "2022-01-01T00:00:00.000", which is correct. I just don't understand what is going wrong. Thanks for your help!
Full code:
const APIbase = "https://api.clockify.me/api/v1"
const APIreportsbase = "https://reports.api.clockify.me/v1"
const myAPIkey = "[HIDDEN FOR PRIVACY]"
const myWorkspaceID = "[HIDDEN FOR PRIVACY]"
function GetHours(userName, startDate, endDate) {
var newJSON = {
"dateRangeStart": startDate,
"dateRangeEnd": endDate,
"summaryFilter": {
"groups": [
"USER"
],
"sortColumn": "GROUP"
}
}
var payload = JSON.stringify (newJSON);
var headers = {"X-Api-Key" : myAPIkey, "content-type" : "application/json"};
var url = APIreportsbase + '/workspaces/' + myWorkspaceID + '/reports/summary'
var options = {
"method": "post",
"contentType": "application/json",
"headers": headers,
"payload": payload,
"muteHttpExceptions" : false
};
var response = UrlFetchApp.fetch(url, options)
var json = response.getContentText();
var data = JSON.parse(json);
var people = data.groupOne;
for (let i = 0; i < people.length; i++) {
if (people[i].name == userName) {
if (people[i].duration == 0) {
return 0;
} else {
return people[i].duration/3600;
}
}
}
}
GetHours();
I got the program working by adding filter so that the second time the program ran, it didn't affect the return value.
if (startDate != null) {
var response = UrlFetchApp.fetch(url, options)
var json = response.getContentText();
.....
}

Google App Script oAuth2 to Questrade API Invalid Token

I'm trying to used Google Apps Script to pull Questrade account information into a Google sheets spreadsheet. I've added the oAuth2 library from GitHub(https://github.com/googleworkspace/apps-script-oauth2) then mostly copy and pasted (with minor edits) from the example code.
The weird thing is this code has worked, exactly how it is, but a day later it no longer works and returns the following:
Exception: Request failed for https://api01.iq.questrade.com returned code 401. Truncated server response: {"code":1017,"message":"Access token is invalid"} (use muteHttpExceptions option to examine full response)
My Google Apps Script is posted below. I've only removed my Questrade Client_ID and Google Script Script_ID. I have three buttons in my spreadsheet which I've linked to functions in the script:
Button 1 - QT oAuth - calls showSidebar
Button 2 - Load Account Info - calls makeRequest
Button 3 - QT Logout - calls logout
Typically, I press the QT Logout button to reset 0Auth2 services then I press the QT oAuth button. This seems to successfully go through the authorization process. I then press the Load Account Info button and about 99 times out of 100 I get the invalid access token message. I don't know if it's relevant, but when I log into Questrades API hub I can see that the script is adding an authorization after the QT oAuth button is pressed but it seems to disappear after about a minute.
The script:
function getQTService() {
// Create a new service with the given name. The name will be used when
// persisting the authorized token, so ensure it is unique within the
// scope of the property store.
return OAuth2.createService('QT')
// Set the endpoint URLs.
.setAuthorizationBaseUrl('https://login.questrade.com/oauth2/authorize')
.setTokenUrl('https://login.questrade.com/oauth2/token')
// Set the client ID and secret.
.setClientId('Client_ID')
.setClientSecret(' ') //there is no client secret but oAuth2 requires one
// Set the name of the callback function in the script referenced
// above that should be invoked to complete the OAuth flow.
.setCallbackFunction('authCallback')
// Set the property store where authorized tokens should be persisted.
.setPropertyStore(PropertiesService.getUserProperties())
.setCache(CacheService.getUserCache())
// Set the scopes to request (space-separated for Google services).
.setScope('read_acc')
}
function showSidebar() {
var QTService = getQTService();
if (!QTService.hasAccess()) {
var authorizationUrl = QTService.getAuthorizationUrl();
var template = HtmlService.createTemplate(
'Authorize. ' +
'Reopen the sidebar when the authorization is complete.');
template.authorizationUrl = authorizationUrl;
var page = template.evaluate();
SpreadsheetApp.getUi().showSidebar(page);
} else {
// ...
}
}
function authCallback(request) {
var QTService = getQTService();
var isAuthorized = QTService.handleCallback(request);
if (isAuthorized) {
return HtmlService.createHtmlOutput('Success! You can close this tab.');
} else {
return HtmlService.createHtmlOutput('Denied. You can close this tab');
}
}
function makeRequest() {
var QTService = getQTService();
var token = QTService.getAccessToken();
var spreadsheet = SpreadsheetApp.openById("Script_ID");
// Get account number
var response = UrlFetchApp.fetch('https://api01.iq.questrade.com/v1/accounts',{
headers: {
Authorization: 'Bearer ' + token
}
});
var json = response.getContentText();
var accountdata = JSON.parse(json);
var j = 0;
while(j < accountdata.accounts.length) {
var Account_num = accountdata.accounts[j].number;
var Account_type = accountdata.accounts[j].type;
var sheet = spreadsheet.getSheetByName(Account_type);
// GET CASH BALANCE
var response = UrlFetchApp.fetch('https://api01.iq.questrade.com/v1/accounts/' + Account_num + '/balances',{
headers: {
Authorization: 'Bearer ' + token
}
});
json = response.getContentText();
var balancedata = JSON.parse(json);
var i = 0;
while(balancedata.perCurrencyBalances[i].currency != 'CAD') {
i=i+1;
}
//send cash value to spreadsheet
sheet.getRange("G1").setValue(balancedata.perCurrencyBalances[i].cash);
// GET POSITIONS
var response = UrlFetchApp.fetch('https://api01.iq.questrade.com/v1/accounts/' + Account_num + '/positions',{
headers: {
Authorization: 'Bearer ' + token
}
});
json = response.getContentText();
var positionsdata = JSON.parse(json);
var num_of_positions = positionsdata.positions.length;
var i = 0;
while(i < num_of_positions) { //this loop is not that smart assumes the positions are where I specify, fix later
if(positionsdata.positions[i].symbol == 'VCN.TO'){
sheet.getRange("D5").setValue(positionsdata.positions[i].openQuantity);
}
if(positionsdata.positions[i].symbol == 'VUN.TO') {
sheet.getRange("D6").setValue(positionsdata.positions[i].openQuantity);
}
if(positionsdata.positions[i].symbol == 'VIU.TO') {
sheet.getRange("D7").setValue(positionsdata.positions[i].openQuantity);
}
if(positionsdata.positions[i].symbol == 'VEE.TO') {
sheet.getRange("D8").setValue(positionsdata.positions[i].openQuantity);
}
i=i+1;
}
j=j+1;
}
//send cash value to spreadsheet
// sheet.getRange("G1").setValue(data.perCurrencyBalances[i].cash);
}
function logout() {
var service = getQTService();
service.reset();
}
Any advice on what might be going wrong here would be greatly appreciated.
I don't think you can rely on using api01. I think you have to extract the api_server from the auth call that gives you a token (or at least I did this using the example on https://www.questrade.com/api/documentation/getting-started with refresh_token). My last 3 refresh_token auths for a bearer have yielded the api06 endpoint. I took your code and with the oauth authorizing and using api06 it works fine.
The magic sauce is var api_server = QTService.getToken().api_server;

Is there a way to limit/throttle how many cells in a Google Sheets column are executing at a time

I don't even know how to word my title. But basically in my GSheets I have a column that has a script and I want to limit how many cells at one time are executing the script. To throttle the executions - because 100 lines of data are imported but the route the script calls can't handle a large load so I end up having a bunch of !ERRORS and have to manually resubmit that data. I added in the script.
function myFunc(value)
{
var serviceUrl = "myUrl" + value;
var response;
try
{
response = UrlFetchApp.fetch(serviceUrl);
}
catch(err)
{
Logger.log(err);
}
finally
{
Logger.log(response.getContentText());
response = JSON.parse(response.getContentText());
var arr = [];
arr.push(response.myValues);
return arr;
}
}
You seem to be running into API quota limits of your requested site.
I would recommend that you stop erroneus data from being pasted in the sheet. By using a try-catch as you are, but tailor it to your needs.
Example:
function myFunc(value) {
var serviceUrl = "yourUrl" + value;
var arr = [];
var response;
try {
// Try something that might throw errors
response = UrlFetchApp.fetch(serviceUrl);
Logger.log(response.getContentText());
response = JSON.parse(response.getContentText());
arr.push(response.myValues);
}
catch(err) {
// Catch any errors gracefully and handle them
Logger.log(err);
}
finally {
// This will always run, so make sure it is something that will not
// put erroneus data in your sheet
if (arr.length > 0){
Logger.log("Response content is: " + response);
}
}
}

Get Google Document as HTML

I had a wild idea that I could build a website blog for an unsophisticated user friend using Google Drive Documents to back it. I was able to create a contentService that compiles a list of documents. However, I can't see a way to convert the document to HTML. I know that Google can render documents in a web page, so I wondered if it was possible to get a rendered version for use in my content service.
Is this possible?
You can try this code :
function getGoogleDocumentAsHTML(){
var id = DocumentApp.getActiveDocument().getId() ;
var forDriveScope = DriveApp.getStorageUsed(); //needed to get Drive Scope requested
var url = "https://docs.google.com/feeds/download/documents/export/Export?id="+id+"&exportFormat=html";
var param = {
method : "get",
headers : {"Authorization": "Bearer " + ScriptApp.getOAuthToken()},
muteHttpExceptions:true,
};
var html = UrlFetchApp.fetch(url,param).getContentText();
Logger.log(html);
}
Node.js Solution
Using the Google APIs Node.js Client
Here's how you can get a google doc as html using google drive's node.js client library.
// import googleapis npm package
var google = require('googleapis');
// variables
var fileId = '<google drive doc file id>',
accessToken = '<oauth access token>';
// oauth setup
var OAuth2 = google.auth.OAuth2,
OAuth2Client = new OAuth2();
// set oauth credentials
OAuth2Client.setCredentials({access_token: accessToken});
// google drive setup
var drive = google.drive({version: 'v3', auth: OAuth2Client});
// download file as text/html
var buffers = [];
drive.files.export(
{
fileId: fileId,
mimeType: 'text/html'
}
)
.on('error', function(err) {
// handle error
})
.on('data', function(data) {
buffers.push(data); // data is a buffer
})
.on('end', function() {
var buffer = Buffer.concat(buffers),
googleDocAsHtml = buffer.toString();
console.log(googleDocAsHtml);
});
Take a look at the Google Drive V3 download docs for more languages and options.
Google docs currently has a function to do this.
Just download to zip(.html) and you can have a zip archive with html & image (if inserted)
I know this is not solution based on code, but its working :)
There is no direct method in GAS to get an HTML version of a doc and this is quite an old enhancement request but the workaround described originally by Henrique Abreu works pretty well, I use it all the time...
The only annoying thing in the authorization process that needs to be called from the script editor which makes it uneasy to use in a shared application (with "script unable" users) but this only happens once ;).
There is also a Library created by Romain Vialard that makes things (a bit) easier... and adds a few other interesting functions.
Here is a little snipped for the new version of goole AOuth following the idea posted by Enrique:
function exportAsHTML(){
var forDriveScope = DriveApp.getStorageUsed(); //needed to get Drive Scope requested
var docID = DocumentApp.getActiveDocument().getId();
var url = "https://docs.google.com/feeds/download/documents/export/Export?id="+docID+"&exportFormat=html";
var param = {
method : "get",
headers : {"Authorization": "Bearer " + ScriptApp.getOAuthToken()},
muteHttpExceptions:true,
};
var html = UrlFetchApp.fetch(url,param).getContentText();
return html;
}
and then use the usual mailApp:
function mailer(){
var docbody = exportAsHTML();
MailApp.sendEmail({
to: "email#mail.com",
subject: "document emailer",
htmlBody: docbody });
}
Hope the new workaround helps
JD
You may use the solution here
/**
* Converts a file to HTML. The Advanced Drive service must be enabled to use
* this function.
*/
function convertToHtml(fileId) {
var file = Drive.Files.get(fileId);
var htmlExportLink = file.exportLinks['text/html'];
if (!htmlExportLink) {
throw 'File cannot be converted to HTML.';
}
var oAuthToken = ScriptApp.getOAuthToken();
var response = UrlFetchApp.fetch(htmlExportLink, {
headers:{
'Authorization': 'Bearer ' + oAuthToken
},
muteHttpExceptions: true
});
if (!response.getResponseCode() == 200) {
throw 'Error converting to HTML: ' + response.getContentText();
}
return response.getContentText();
}
Pass as fileId, the id of the google doc and to enable advanced drive services follow the instructions here.
I've had this problem as well. The HTML that the Document HTML Export spits out is really ugly, so this was my solution:
/**
* Takes in a Google Doc ID, gets that doc in HTML format, cleans up the markup, and returns the resulting HTML string.
*
* #param {string} the id of the google doc
* #param {boolean} [useCaching] enable or disable caching. default true.
* #return {string} the doc's body in html format
*/
function getContent(id, useCaching) {
if (!id) {
throw "Please call this API with a valid Google Doc ID";
}
if (useCaching == null) {
useCaching = true;
}
if (typeof useCaching != "boolean") {
throw "If you're going to specify useCaching, it must be boolean.";
}
var cache = CacheService.getScriptCache();
var cached = cache.get(id); // see if we have a cached version of our parsed html
if (cached && useCaching) {
var html = cached;
Logger.log("Pulling doc html from cache...");
} else {
Logger.log("Grabbing and parsing fresh html from the doc...");
try {
var doc = DriveApp.getFileById(id);
} catch (err) {
throw "Please call this API with a valid Google Doc ID. " + err.message;
}
var docName = doc.getName();
var forDriveScope = DriveApp.getStorageUsed(); // needed to get Drive Scope requested in ScriptApp.getOAuthToken();
var url = "https://docs.google.com/feeds/download/documents/export/Export?id=" + id + "&exportFormat=html";
var param = {
method: "get",
headers: {"Authorization": "Bearer " + ScriptApp.getOAuthToken()},
muteHttpExceptions:true,
};
var html = UrlFetchApp.fetch(url, param).getContentText();
// nuke the whole head section, including the stylesheet and meta tag
html = html.replace(/<head>.*<\/head>/, '');
// remove almost all html attributes
html = html.replace(/ (id|class|style|start|colspan|rowspan)="[^"]*"/g, '');
// remove all of the spans, as well as the outer html and body
html = html.replace(/<(span|\/span|body|\/body|html|\/html)>/g, '');
// clearly the superior way of denoting line breaks
html = html.replace(/<br>/g, '<br />');
cache.put(id, html, 900) // cache doc contents for 15 minutes, in case we get a lot of requests
}
Logger.log(html);
return html;
}
https://gist.github.com/leoherzog/cc229d14a89e6327336177bb07ac2980
Perhaps this would work for you...
function doGet() {
var blob = DriveApp.getFileById('myFileId').getAsHTML();
return HtmlService.createHtmlOutput(blob);
}