Bandwidth quota exceeded. Limit the data transfer speed - google-apps-script

I received the following error on a script that we run time-based.
Exception: Bandwidth quota exceeded: https://app.enzyme.finance/api/vault-performance?vault=0x95fca2e84556443c1bc0c6416ee17be0e6844cd0&currency=eur&network=ethereum. Limit the data transfer speed.
I have no clue which quota I'm exceeding. So how fixing this issue is quite a mystery.
In the code below I'm doing the following:
Delete any existing triggers deleteTriggers()
Schedule a trigger via function on time: 00:00 scheduledTrigger(0,0)
Fetch data from URL and parse that data into Google Sheets function_Triggered()
In run this code time-driven day timer at: 11 pm to midnight.
The code below:
function setTrigger() {
deleteTriggers();
scheduledTrigger(0,0);
}
function scheduledTrigger(hours,minutes){
var today_D = new Date();
var year = today_D.getFullYear();
var month = today_D.getMonth();
var day = today_D.getDate();
pars = [year,month,day,hours,minutes];
var scheduled_D = new Date(...pars);
scheduled_D.setDate(scheduled_D.getDate() + 1);
var hours_remain=Math.abs(scheduled_D - today_D) / 36e5;
ScriptApp.newTrigger("function_Triggered")
.timeBased()
.after(hours_remain * 60 * 60 * 1000)
.create()
}
function deleteTriggers() {
var triggers = ScriptApp.getProjectTriggers();
for (var i = 0; i < triggers.length; i++) {
if ( triggers[i].getHandlerFunction() == "function_Triggered") {
ScriptApp.deleteTrigger(triggers[i]);
}
}
}
function function_Triggered() {
var ss = SpreadsheetApp.getActiveSpreadsheet();
var raw = ss.getSheetByName('raw');
var current = ss.getSheetByName('nav.current');
var database = ss.getSheetByName('nav.database');
var url = "https://app.enzyme.finance/api/vault-performance?vault=0x95fca2e84556443c1bc0c6416ee17be0e6844cd0&currency=eur&network=ethereum";
var response = UrlFetchApp.fetch(url); // get feed
var dataAll = JSON.parse(response.getContentText());
var rows = [Object.keys(dataAll)]; // Retrieve headers.
var temp = [];
for (var i = 0; i < rows[0].length; i++) {
temp.push(dataAll[rows[0][i]]); // Retrieve values.
}
rows.push(temp);
raw.getRange(1,1,rows.length,rows[0].length).setValues(rows); // Put values to Spreadsheet.
// count rows to snap
var current_rows = current.getLastRow();
var database_rows = database.getLastRow() + 1;
var rows_new = current.getRange("A2:B" + current_rows).getValues();
// snap rows, can run this on a trigger to be timed
database.getRange("A" + database_rows + ":B" + database_rows).setValues(rows_new);
}

Related

Speed up apps script from MySQL Query Google Sseets

I am facing an issue like many before with regards to a timeout out Google Apps Script, I am reading the data from a indexed/persisted table in a MySQL Database, the table in question has 71 columns and a total of 28000 rows, the sheet in google sheets I am writing to has no calculations etc on it which might slow things down - those happen on other sheets.
Please can you review the below that I am using and propose any changes to assist in avoiding the time out?
var server = 'xx.xx.xx.xxx';
var port = xxxx;
var dbName = 'test';
var username = 'test';
var password = 'xxx';
var url = 'jdbc:mysql://'+server+':'+port+'/'+dbName;
function readDataPast() {
var conn = Jdbc.getConnection(url, username, password);
var stmt = conn.createStatement();
var results = stmt.executeQuery('SELECT * FROM test.test_table');
var metaData = results.getMetaData();
var numCols = metaData.getColumnCount();
var spreadsheet = SpreadsheetApp.getActive();
var sheet = spreadsheet.getSheetByName('Raw_Data');
sheet.clearContents();
var arr = [];
let row = [];
for (var col = 0; col < numCols; col++) {
row.push(metaData.getColumnName(col + 1));
}
arr.push(row);
while (results.next()) {
row = [];
for (var col = 0; col < numCols; col++) {
row.push(results.getString(col + 1));
}
arr.push(row)
}
sheet.getRange(1, 1, arr.length, arr[0].length).setValues(arr);
results.close();
stmt.close();
}
Issue:
I don't think the script can be made considerably faster, since potential improvements (e.g. using Sheets API as suggested by Ninca Tirtil) don't affect significatively the bulk of the script (iterating through 28000 rows).
Workaround:
Therefore, instead of trying to speed it up, I'd suggest accomplishing this in multiple executions. To that goal, I'd do the following:
Check execution time after each iteration. If this time is close to the time limit, end the loop and write current data to the sheet. You can use the Date object for this.
Create the following time-based trigger at the end of your function: after(durationMilliseconds). Thanks to this, the function will fire automatically after the amount of milliseconds you indicate. After each execution, a trigger will be created to fire the next execution.
Because you want to split the loop, you have to store the row index somewhere (you could use PropertiesService at the end of each execution, for example) and retrieve it at the beginning of the next, so that in each successive execution, the script resumes the loop where it left it. You can get the row index via getRow(), and then move to that row in the next execution via relative(rows).
Code sample:
var maxTimeDiff = 1000 * 60 * 5; // 5 minutes
const PROPERTY_KEY = "Row index";
function setRowIndex(rowIndex) {
const scriptProps = PropertiesService.getScriptProperties();
scriptProps.setProperty(PROPERTY_KEY, rowIndex);
}
function getRowIndex() {
const scriptProps = PropertiesService.getScriptProperties();
const rowIndex = scriptProps.getProperty(PROPERTY_KEY);
return rowIndex;
}
function createTrigger() {
ScriptApp.newTrigger("readDataPast")
.timeBased()
.after(60 * 1000) // Next execution after a minute
.create();
}
function readDataPast() {
var startTime = new Date();
var conn = Jdbc.getConnection(url, username, password);
var stmt = conn.createStatement();
var results = stmt.executeQuery('SELECT * FROM test.test_table');
var spreadsheet = SpreadsheetApp.getActive();
var sheet = spreadsheet.getSheetByName('Raw_Data');
var rowIndex = getRowIndex();
var arr = [];
let row = [];
if (!rowIndex || rowIndex == 0) { // Clear sheet and add metadata if first execution
sheet.clearContents();
var metaData = results.getMetaData();
var numCols = metaData.getColumnCount();
for (var col = 0; col < numCols; col++) {
row.push(metaData.getColumnName(col + 1));
}
arr.push(row);
} else {
results.relative(rowIndex); // Move to current row
}
while (results.next()) {
row = [];
for (var col = 0; col < numCols; col++) {
row.push(results.getString(col + 1));
}
arr.push(row);
if (new Date() - startTime > maxTimeDiff) break; // End iteration if long time
}
var currentRow = results.getRow(); // 0 if all rows have been iterated
setRowIndex(currentRow);
var lastRow = sheet.getLastRow();
sheet.getRange(lastRow + 1, 1, arr.length, arr[0].length).setValues(arr);
results.close();
stmt.close();
if (currentRow) createTrigger(); // Create trigger if iteration is not finished
}

Filter gmails using timestamp

Am new dabbling with Google Apps Script; would like to ask if I'm in the right direction, and how to I manipulate time within the script.
I'm struggling in trying to maniuplate time values in Google App Script, basically I am able to pull the timestamp of each email sent, but I only want to paste into the spreadsheet email information that were recent, e.g. within 30minutes from script run time. This is to avoid pulling duplicate information.
Not sure if there is a currentTime() function here, or I have to create a new Date() object and do some calculations from there. Tried a few variations and nothing seemed to work proper.
Would appreciate any help in getting towards the right direction in doing this thank you!
function getDetails(){
var DEST_URL = "SHEET_URL"; //redacted for sensitivity
var DEST_SHEETNAME = "Test";
var destss = SpreadsheetApp.openByUrl(DEST_URL);
var destSheet = destss.getSheetByName(DEST_SHEETNAME);
var threads = GmailApp.search("FILTERS"); //filter settings redacted for sensitivity
for(var i = 0; i < threads.length; i++){
var messages=threads[i].getMessages();
for(var j =0; j < 1; j++){ //only take first message in thread
var message = messages[j];
var subject = message.getSubject() ;
var sentTimeStamp = message.getDate();
if(sentTimeStamp is within last 30minutes as of script run time){ //this is where i need help
var delimitString = subject.split("is sent");
var detailName = delimitString[0];
var lastRow = destSheet.getLastRow();
destSheet.getRange(lastRow + 1,1).setValue(detailName);
destSheet.getRange(lastRow + 1,2),setValue(sentTimeStamp);
}
}
}
}
You can convert timeStamp into ms seconds and then compare to the value of "30 s ago"
Sample:
var sentTimeStamp = message.getDate();
var now = new Date();
var ThirtyMinutesAgo = now-30*60*1000;
if(sentTimeStamp.getTime() < ThirtyMinutesAgo){
...
}
References:
newDate()
getTime()
Another idea would be to query for emails that you received the last 30 minutes.
Explanation:
You can get the emails that you received the last 30 minutes ago as a query in the GmailApp.search function. See this link to see what filters you can use.
This will get the last emails with keyword "FILTERS" that you received the last 30 minutes.
var ThirtyMinutesAgo = new Date();
ThirtyMinutesAgo.setMinutes(ThirtyMinutesAgo.getMinutes() - 30);
const queryString = `"FILTERS" newer:${Math.round(ThirtyMinutesAgo.getTime()/1000)}`
const threads = GmailApp.search(queryString); // threads the last 30 minutes
This approach is more efficient for two reasons:
You have less data (threads) to iterate over with the for loop.
You don't need to apply and if statement on every thread.
Solution:
function getDetails(){
var DEST_URL = "SHEET_URL"; //redacted for sensitivity
var DEST_SHEETNAME = "Test";
var destss = SpreadsheetApp.openByUrl(DEST_URL);
var destSheet = destss.getSheetByName(DEST_SHEETNAME);
// var threads = GmailApp.search("FILTERS"); //filter settings redacted for sensitivity
// new code
var ThirtyMinutesAgo = new Date();
ThirtyMinutesAgo.setMinutes(ThirtyMinutesAgo.getMinutes() - 30);
const queryString = `"FILTERS" newer:${Math.round(ThirtyMinutesAgo.getTime()/1000)}`
const threads = GmailApp.search(queryString); // threads the last 30 minutes
//
for(var i = 0; i < threads.length; i++){
var messages=threads[i].getMessages();
for(var j =0; j < 1; j++){ //only take first message in thread
var message = messages[j];
var subject = message.getSubject() ;
var sentTimeStamp = message.getDate();
var delimitString = subject.split("is sent");
var detailName = delimitString[0];
var lastRow = destSheet.getLastRow();
destSheet.getRange(lastRow + 1,1).setValue(detailName);
destSheet.getRange(lastRow + 1,2),setValue(sentTimeStamp);
}
}
}
}

Google script exceeded maximum execution time work around [duplicate]

This question already exists:
Exceeded maximum execution time in Google Apps Script [duplicate]
Closed 2 years ago.
connecting to a sql server database via jdbc from a google sheets doc. This table has some 30k odd records and hit "exceeded maximum execution time" error. Is there a way to improve performance or a work around to the 6 minute execution time.
function myFunction() {
var conn = Jdbc.getConnection("jdbc:sqlserver://dbURL", "username", "password");
var stmt = conn.createStatement();
stmt.setMaxRows(3000);
var start = new Date();
var rs = stmt.executeQuery('SELECT * FROM dbo.myTable');
var doc = SpreadsheetApp.getActiveSpreadsheet();
var cell = doc.getRange('a1');
var row = 0;
var data = [];
while (rs.next()) {
var rowData = [];
for (var col = 0; col < rs.getMetaData().getColumnCount(); col++) {
rowData[col] = (rs.getString(col + 1));
}
data[row] = rowData;
row++;
}
rs.close();
stmt.close();
conn.close();
var end = new Date();
Logger.log('Time elapsed: ' + (end.getTime() - start.getTime()));
}
Thank you.
Modified the code to use Continuous Batch Library, even though I don't get Exceeded execution time error looks like its getting into an infinite loop and writing not one records to the sheets.
var FUNCTION_NAME = "test";
var EMAIL_RECIPIENT = "";
function test() {
ContinuousBatchLibrary.startOrResumeContinousExecutionInstance(FUNCTION_NAME)
var conn = Jdbc.getConnection("jdbc:sqlserver://HOST:PORT/DBNAME","user","password");
var stmt = conn.createStatement();
stmt.setMaxRows(100);
var rs = stmt.executeQuery('SELECT * FROM myTable');
var SS;
var Sheet;
var SheetRange;
// OPEN EXISTING SPREADSHEET
SS = SpreadsheetApp.openById(SpreadsheetApp.getActiveSpreadsheet().getId());
// SET SPREADSHEET ACTIVE
SpreadsheetApp.setActiveSpreadsheet(SS);
Sheet = SpreadsheetApp.setActiveSheet(SS.getSheetByName(SpreadsheetApp.getActiveSpreadsheet().getSheetName()));
// GET NUMBER OF COLUMNS
var ColCount = rs.getMetaData().getColumnCount();
// SET COLUMN HEADERS
for (var col = 1; col <= ColCount; col++) {
GSheet.getRange(1, col).setValue(rs.getMetaData().getColumnName(col));
}
// SET RANGE TO FIRST ROW AND BOLD
GSheetRange = GSheet.getRange(1,1,1,GSheet.getLastColumn());
GSheetRange.setFontWeight("bold");
var doc = SpreadsheetApp.getActiveSpreadsheet();
var cell = doc.getRange('a2');
var row = 0;
var data = [];
var i = ContinuousBatchLibrary.getBatchKey(FUNCTION_NAME) || 0;
for (; i < row.length; i++) {
while (rs.next()) {
var rowData = [];
for (var col = 0; col < rs.getMetaData().getColumnCount(); col++) {
rowData[col] = (rs.getString(col + 1));
}
data[row] = rowData;
row++;
}
if (ContinuousBatchLibrary.isTimeRunningOut(FUNCTION_NAME)) {
ContinuousBatchLibrary.setBatchKey(FUNCTION_NAME, i)
break;
}
}
rs.close();
stmt.close();
conn.close();
if (i === row.length) {
ContinuousBatchLibrary.endContinuousExecutionInstance(FUNCTION_NAME, EMAIL_RECIPIENT, "task complete")
}
}
This answer is probably still the best work around for the 6 min timer.
The cumulative trigger run time limit is one hour a day so if you are getting <10% of the way thru your data before the timeout you will run into that as well.

Import Google or iCal calendar data into a google spreadsheet

I am trying to keep a record of google calendar entries in a google spreadsheet to further process the data. I have the following code, which I borrowed from other sources:
function importEvents(){
var startOfDay = new Date();
startOfDay.setUTCHours(0);
startOfDay.setMinutes(0);
startOfDay.setSeconds(0);
startOfDay.setMilliseconds(0);
var endOfDay = new Date(startOfDay.getTime() + 24 * 60 * 60 * 1000);
var Calendar = CalendarApp.getCalendarById("[calendarIDhere]");
var events = Calendar.getEvents(startOfDay, endOfDay)
var events_sheet = SpreadsheetApp.getActiveSpreadsheet().getSheetByName("ImportedEvents");
var lr = events_sheet.getLastRow();
var eventarray = new Array();
var i = 0; // edited
for (i = 0; i < events.length; i++) {
line = new Array();
line.push(events[i].getTitle());
line.push(events[i].getDescription());
line.push(events[i].getStartTime());
line.push(events[i].getEndTime());
eventarray.push(line);
}
events_sheet.getRange('A1:D' + (lr)).setValues(eventarray);
var l = events_sheet.getLastRow();
var m = events_sheet.getMaxRows();
events_sheet.deleteRows(l+1,m-l);
}
Getting errors with the .getRange() function regarding incorrect height of data
If you're setting values on a range, your range has to match the size of the values array in both columns and rows.
Instead of the following line:
events_sheet.getRange('A1:D' + (lr)).setValues(eventarray);
Try this:
events_sheet.getRange('A1:D' + eventarray.length).setValues(eventarray);

calendar from apps script : every time authorized required. Only work with primary calendar

In a spreadsheet, I have a app script for count hours in a google calendar and the output is copied in the spreadsheet.
A few days ago, anything worked fine.
but today (monday July 1 2013 ), when I try run the script, every time, I get the message "Authorized required".
http://cl.ly/Q0bd
I press in "Authorized" button, and re-run, and again get the message "Authorized required".
the code in a gist
// add menu
function onOpen() {
var ss = SpreadsheetApp.getActiveSpreadsheet();
var menuEntries = [{name:"Calcular Horas", functionName: "calculateHours"}];
ss.addMenu("Hours", menuEntries);
// calcular al iniciar
//calculateHours();
}
function authorize() {
var oauthConfig = UrlFetchApp.addOAuthService("calendar");
var scope = "https://www.googleapis.com/auth/calendar";
oauthConfig.setConsumerKey("anonymous");
oauthConfig.setConsumerSecret("anonymous");
oauthConfig.setRequestTokenUrl("https://www.google.com/accounts/OAuthGetRequestToken?scope="+scope);
oauthConfig.setAuthorizationUrl("https://accounts.google.com/OAuthAuthorizeToken");
oauthConfig.setAccessTokenUrl("https://www.google.com/accounts/OAuthGetAccessToken");
}
/*
* Count hours of events with same name
*/
function countHours(calId, eventName){
authorize();
var cal = CalendarApp.getCalendarById(calId);
var key = "...";
var query = encodeURIComponent(eventName);
calId = encodeURIComponent(calId);
var params = {
method: "get",
oAuthServiceName: "calendar",
oAuthUseToken: "always",
};
var url = "https://www.googleapis.com/calendar/v3/calendars/"+
calId+"/events?q=" + query + "&key=" + key;
var request = UrlFetchApp.fetch(url, params);
//Logger.log(url);
var response = Utilities.jsonParse(request.getContentText());
var items = response.items;
var start, end;
var hours = 0;
for ( i = 0 ; i < items.length ; i++){
if ( items[i].status != "cancelled" ){
if ( items[i].summary == eventName ){
start = items[i].start.dateTime;
end = items[i].end.dateTime;
start = new Date(start.replace(/-/g,'/').replace(/[A-Z]/,' ').substr(0,19) );
end = new Date(end.replace(/-/g,'/').replace(/[A-Z]/,' ').substr(0,19));
hours = hours + ( end - start ) / ( 1000 * 60 * 60 );
}
}
}
return hours;
}
function calculateHours(){
var ss = SpreadsheetApp.getActiveSpreadsheet();
var s = ss.getSheets()[0];
var rows = s.getDataRange();
var nRows = rows.getNumRows();
var values = rows.getValues();
// from second row
for ( var i = 1; i < nRows ; i ++){
var row = values[i];
var h = countHours(row[0], row[1]);
s.getRange(i+1, 3).setValue(h);
}
}
EDIT
When I change the line
var url = "https://www.googleapis.com/calendar/v3/calendars/"+
calId+"/events?q=" + query + "&key=" + key;
to
var url = "https://www.googleapis.com/calendar/v3/calendars/"+
"primary"+"/events?q=" + query + "&key=" + key;
this work, but is only valid for the primary calendar.
finally I changed the access to calendar API to CalendarApp service with .getEvents
var cal = CalendarApp.getCalendarById(cal_id);
var this_year = new Date(2013,0,1);
var now = new Date()
var events = cal.getEvents(this_year, now, {search: event_name});