Google Script, Run functions in sequence without exceeding execution time - google-apps-script

I have a lot of functions that fetch JSON API data from a website, but if I run them in sequence in this way, I get the exceeding execution time error:
function fetchdata () {
data1();
data2();
data3();
data4();
...
}
I can schedule a trigger to run them at 5 minutes one of the other (cause a single one runs in 3 minutes), but I would like to know if there is any other way around. Thank you
EDIT:
Every "data" function is like this one:
function data1() {
var addresses = SpreadsheetApp.getActiveSpreadsheet().getSheetByName("Import");
var baseUrl = 'https://myapiurl';
var address = addresses.getRange(2, 1, 500).getValues();
for(var i=0;i<address.length;i++){
var addrID = address[i][0];
var url = baseUrl.concat(addrID);
var responseAPI = UrlFetchApp.fetch(url);
var json = JSON.parse(responseAPI.getContentText());
var data = [[json.result]];
var dataRange = addresses.getRange(i+2, 2).setValue(data);
}
}
data2 is for rows 502-1001,
data3 is for rows 1002-1501,
and so on...

I just removed the concat because it has performance issues according to MDN but obviously the real problem is the fetch and there's not much we can do about that unless you can get your external api to dump a bigger batch.
You could initiate each function from a webapp and then have it return via withSuccessHandler and then start the next script in the series and daisy chain your way through all of the subfunctions until your done. Each sub function will take it's 3 minutes or so but you only have to worry about keeping each sub function under 6 minutes that way.
function data1()
{
var addresses = SpreadsheetApp.getActiveSpreadsheet().getSheetByName("Import");
var baseUrl = 'https://myapiurl';
var address = addresses.getRange(2, 1, 500).getValues();
for(var i=0;i<address.length;i++){
var responseAPI = UrlFetchApp.fetch(baseUrl + address[i][0]);
var json = JSON.parse(responseAPI.getContentText());
var data = [[json.result]];
var dataRange = addresses.getRange(i+2, 2).setValue(data);
}
}

Related

Apps Script custom function working in script editor but not in Google Sheet custom function

I have built a simple custom function in Apps Script using URLFetchApp to get the follower count for TikTok accounts.
function tiktok_fans() {
var raw_data = new RegExp(/("followerCount":)([0-9]+)/g);
var handle = '#charlidamelio';
var web_content = UrlFetchApp.fetch('https://www.tiktok.com/'+ handle + '?lang=en').getContentText();
var match_text = raw_data.exec(web_content);
var result = (match_text[2]);
Logger.log(result)
return result
}
The Log comes back with the correct number for followers.
However, when I change the code to;
function tiktok_fans(handle) {
var raw_data = new RegExp(/("followerCount":)([0-9]+)/g);
//var handle = '#charlidamelio';
var web_content = UrlFetchApp.fetch('https://www.tiktok.com/'+ handle + '?lang=en').getContentText();
var match_text = raw_data.exec(web_content);
var result = (match_text[2]);
Logger.log(result)
return result
}
and use it in a spreadsheet for example =tiktok_fans(A1), where A1 has #charlidamelio I get an #ERROR response in the cell
TypeError: Cannot read property '2' of null (line 6).
Why does it work in the logs but not in the spreadsheet?
--additional info--
Still getting the same error after testing #Tanaike answer below, "TypeError: Cannot read property '2' of null (line 6)."
Have mapped out manually to see the error, each time the below runs, a different log returns "null". I believe this is to do with the ContentText size/in the cache. I have tried utilising Utilities.sleep() in between functions with no luck, I still get null's.
code
var raw_data = new RegExp(/("followerCount":)([0-9]+)/g);
//tiktok urls
var qld = UrlFetchApp.fetch('https://www.tiktok.com/#thisisqueensland?lang=en').getContentText();
var nsw = UrlFetchApp.fetch('https://www.tiktok.com/#visitnsw?lang=en').getContentText();
var syd = UrlFetchApp.fetch('https://www.tiktok.com/#sydney?lang=en').getContentText();
var tas = UrlFetchApp.fetch('https://www.tiktok.com/#tasmania?lang=en').getContentText();
var nt = UrlFetchApp.fetch('https://www.tiktok.com/#ntaustralia?lang=en').getContentText();
var nz = UrlFetchApp.fetch('https://www.tiktok.com/#purenz?lang=en').getContentText();
var aus = UrlFetchApp.fetch('https://www.tiktok.com/#australia?lang=en').getContentText();
var vic = UrlFetchApp.fetch('https://www.tiktok.com/#visitmelbourne?lang=en').getContentText();
//find folowers with regex
var match_qld = raw_data.exec(qld);
var match_nsw = raw_data.exec(nsw);
var match_syd = raw_data.exec(syd);
var match_tas = raw_data.exec(tas);
var match_nt = raw_data.exec(nt);
var match_nz = raw_data.exec(nz);
var match_aus = raw_data.exec(aus);
var match_vic = raw_data.exec(vic);
Logger.log(match_qld);
Logger.log(match_nsw);
Logger.log(match_syd);
Logger.log(match_tas);
Logger.log(match_nt);
Logger.log(match_nz);
Logger.log(match_aus);
Logger.log(match_vic);
Issue:
From your situation, I remembered that the request of UrlFetchApp with the custom function is different from the request of UrlFetchApp with the script editor. So I thought that the reason for your issue might be related to this thread. https://stackoverflow.com/a/63024816 In your situation, your situation seems to be the opposite of this thread. But, it is considered that this issue is due to the specification of the site.
In order to check this difference, I checked the file size of the retrieved HTML data.
The file size of HTML data retrieved by UrlFetchApp executing with the script editor is 518k bytes.
The file size of HTML data retrieved by UrlFetchApp executing with the custom function is 9k bytes.
It seems that the request of UrlFetchApp executing with the custom function is the same as that of UrlFetchApp executing withWeb Apps. The data of 9k bytes are retrieved by using this.
From the above result, it is found that the retrieved HTML is different between the script editor and the custom function. Namely, the HTML data retrieved by the custom function doesn't include the regex of ("followerCount":)([0-9]+). By this, such an error occurs. I thought that this might be the reason for your issue.
Workaround:
When I tested your situation with Web Apps and triggers, the same issue occurs. By this, in the current stage, I thought that the method for automatically executing the script might not be able to be used. So, as a workaround, how about using a button and the custom menu? When the script is run by the button and the custom menu, the script works. It seems that this method is the same as that of the script editor.
The sample script is as follows.
Sample script:
Before you run the script, please set range. For example, please assign this function to a button on Spreadsheet. When you click the button, the script is run. In this sample, it supposes that the values like #charlidamelio are put to the column "A".
function sample() {
var range = "A2:A10"; // Please set the range of "handle".
var raw_data = new RegExp(/("followerCount":)([0-9]+)/g);
var sheet = SpreadsheetApp.getActiveSheet();
var r = sheet.getRange(range);
var values = r.getValues();
var res = values.map(([handle]) => {
if (handle != "") {
var web_content = UrlFetchApp.fetch('https://www.tiktok.com/'+ handle + '?lang=en').getContentText();
var match_text = raw_data.exec(web_content);
return [match_text[2]];
}
return [""];
});
r.offset(0, 1).setValues(res);
}
When this script is run, the values are retrieved from the URL and put to the column "B".
Note:
This is a simple script. So please modify it for your actual situation.
Reference:
Related thread.
UrlFetchApp request fails in Menu Functions but not in Custom Functions (connecting to external REST API)
Added:
About the following additional question,
whilst this works for 1 TikTok handle, when trying to run a list of multiple it fails each time, with the error TypeError: Cannot read property '2' of null. After doing some investigating and manually mapping out 8 handles, I can see that each time it runs, it returns "null" for one or more of the web_content variables. Is there a way to slow the script down/run each UrlFetchApp one at a time to ensure each returns content?
i've tried this and still getting an error. Have tried up to 10000ms. I've added some more detail to the original question, hope this makes sense as to the error. It is always in a different log that I get nulls, hence why I think it's a timing or cache issue.
In this case, how about the following sample script?
Sample script:
In this sample script, when the value cannot be retrieved from the URL, the value is tried to retrieve again as the retry. This sample script uses the 2 times as the retry. So when the value cannot be retrieved by 2 retries, the empty value is returned.
function sample() {
var range = "A2:A10"; // Please set the range of "handle".
var raw_data = new RegExp(/("followerCount":)([0-9]+)/g);
var sheet = SpreadsheetApp.getActiveSheet();
var r = sheet.getRange(range);
var values = r.getValues();
var res = values.map(([handle]) => {
if (handle != "") {
var web_content = UrlFetchApp.fetch('https://www.tiktok.com/'+ handle + '?lang=en').getContentText();
var match_text = raw_data.exec(web_content);
if (!match_text || match_text.length != 3) {
var retry = 2; // Number of retry.
for (var i = 0; i < retry; i++) {
Utilities.sleep(3000);
web_content = UrlFetchApp.fetch('https://www.tiktok.com/'+ handle + '?lang=en').getContentText();
match_text = raw_data.exec(web_content);
if (match_text || match_text.length == 3) break;
}
}
return [match_text && match_text.length == 3 ? match_text[2] : ""];
}
return [""];
});
r.offset(0, 1).setValues(res);
}
Please adjust the value of retry and Utilities.sleep(3000).
This works for me as a Custom Function:
function MYFUNK(n=2) {
const url = 'my website url'
const re = new RegExp(`<p id="un${n}.*\/p>`,'g')
const r = UrlFetchApp.fetch(url).getContentText();
const v = r.match(re);
Logger.log(v);
return v;
}
I used my own website and I have several paragraphs with ids from un1 to un7 and I'm taking the value of A1 for the only parameter. It returns the correct string each time I change it.

Code shortening for data entry form Google App script

I'm new to coding, and I know I am going a long way about this and making my script run slow, but I can't figure out how to shorten and optimise it (now I have tried to map the second bit of code using Marios comment)
I have made a data entry form on Google Sheets for athletes I coach to use as a training diary. After recording training data in a session, they hit the save button and this script transfers it to a different spreadsheet with all of their training data ever in.
Below is a section of code I have attempted to shorten with Marios comment:
function submitSession1() {
workloadSubmit();
myValue();
}
function workloadSubmit(){
var inputSS = SpreadsheetApp.getActiveSpreadsheet();
var inputS = inputSS.getSheetByName("Session 1");
var outputSS = SpreadsheetApp.openByUrl()
var workloadS = outputSS.getSheetByName();
var dtCurrentTime = new Date();
//Input Values for Workload data
var workloads = [[inputS.getRange("M1").getValue(),
inputS.getRange("N1").getValue(),
inputS.getRange("O1").getValue(),
inputS.getRange("P1").getValue(),
inputS.getRange("AK3").getValue(),
inputS.getRange("AK5").getValue(),
inputS.getRange("AL3").getValue(),
inputS.getRange("AL5").getValue(),
inputS.getRange("BC3").getValue(),
inputS.getRange("BC5").getValue(),
inputS.getRange("BD3").getValue(),
inputS.getRange("BD5").getValue(),
inputS.getRange("AM3").getValue(),
inputS.getRange("AM5").getValue(),
inputS.getRange("AN3").getValue(),
inputS.getRange("AN5").getValue(),
inputS.getRange("AO3").getValue(),
inputS.getRange("AO5").getValue(),
inputS.getRange("AP3").getValue(),
inputS.getRange("AP5").getValue(),
inputS.getRange("AQ3").getValue(),
inputS.getRange("AQ5").getValue(),
inputS.getRange("AR3").getValue(),
inputS.getRange("AR5").getValue(),
inputS.getRange("AS3").getValue(),
inputS.getRange("AS5").getValue(),
inputS.getRange("AT3").getValue(),
inputS.getRange("AT5").getValue(),
inputS.getRange("AU3").getValue(),
inputS.getRange("AU5").getValue(),
inputS.getRange("AV3").getValue(),
inputS.getRange("AV5").getValue(),
inputS.getRange("AW3").getValue(),
inputS.getRange("AW5").getValue(),
inputS.getRange("AX3").getValue(),
inputS.getRange("AX5").getValue(),
inputS.getRange("AY3").getValue(),
inputS.getRange("AY5").getValue(),
inputS.getRange("AZ3").getValue(),
inputS.getRange("AZ5").getValue(),
inputS.getRange("BA3").getValue(),
inputS.getRange("BA5").getValue(),
inputS.getRange("BB3").getValue(),
inputS.getRange("BB5").getValue(),
dtCurrentTime]];
workloadS.getRange(workloadS.getLastRow()+1, 1, 1,
45).setValues(workloads);
}
// Drills Data Submit
function myValue(col) {
var inputSS = SpreadsheetApp.getActiveSpreadsheet();
var inputS = inputSS.getSheetByName("Session 1");
var outputSS = SpreadsheetApp.openByUrl()
var drillsS = outputSS.getSheetByName("Drills Data");
var dtCurrentTime = new Date();
return inputS.getRange(col).getValue();
}
var colns = ["M1", "N1", "O1", "P1", "A14","B14","D14","F14","G14","H14","J14","K14","L14","M14","N14","O14","P14","Q14","R14","S14","T14","U14"];
var drillsData = colns.map(myValue)
drillsData.push(dtCurrentTime)
I am now getting the error code:
Exception: Argument cannot be null: a1Notation (line 71, file "Code")Dismiss
Any help is much appreciated
You can calculate drillsData using maps:
function myValue(col) {
return inputS.getRange(col).getValue();
}
var colns= ["M1", "N1", "O1", "P1", "A14","B14","D14","F14","G14","H14","J14",
"K14","L14","M14","N14","O14","P14","Q14","R14","S14","T14","U14"];
var drillsData = colns.map(myValue)
drillsData.push(dtCurrentTime)
*Don't forget to call drillsData as [drillsData].
Unfortunately, the columns you want to retrieve are not sequential, therefore selecting the full range is not an option.
Or you can create custom functions to make your code look cleaner:
function importSheets(sheetN) {
return outputSS.getSheetByName(sheetN);
}
var workloadS = importSheets("W.L + Full Routine Data")
For the latter you can again create maps using the same logic described for one.
As a result, you can have a collection of sheets objects as elements in an array and call by using their index.

Google Scripts - View Log stuck at "Waiting for logs, please wait..."

I'm trying to run a script for my Google Sheet on Scripts, and a function isn't working properly. I have some loggers in place to check why this is happening, but anytime I try to open the Logs tab, I get this:
... and it's just stuck there forever.
Has anyone ever had this problem? Any potential fixes? Thanks
EDIT: My executions window looks like so:
EDIT 2: Here is the code I'm trying to run, with segment = 1. SPREADSHEETS is just a variable that I'm unfortunately not able to share, but it just contains some import segment information that directs to either 1 or 2.
function CopyPasteAllSheets(segment) {
for (x in SPREADSHEETS) {
if (SPREADSHEETS[x].IMPORTSEGMENT != segment) {
// DRR added app which is redundant to intakeSpreadhseet, but keeps logic more readable
app.toast('running loop')
console.log("ID: " + SPREADSHEETS[x].SOURCE.ID + "NO MATCH");
} else {
// Logger.log("x: "+ x) // keep commented out
var intakeSpreadsheet = SpreadsheetApp.openById(SPREADSHEETS[x].INTAKE.ID);
var intakeSheet = intakeSpreadsheet.getSheetByName(SPREADSHEETS[x].INTAKE.SHEET); //confirm formatting conventions
// This is functionally equivlent to the above, except we don't have a reference to intakeSpreadsheet anymore
// Access the Spreadsheet and sheet you want to copy the data TO
console.log("ID: "+ SPREADSHEETS[x].SOURCE.ID)
var sourceSpreadsheet = SpreadsheetApp.openById(SPREADSHEETS[x].SOURCE.ID);
var sourceSheet = sourceSpreadsheet.getSheetByName(SPREADSHEETS[x].SOURCE.SHEET);
var sourceStartRow = SPREADSHEETS[x].SOURCE.STARTROW;
var sourceStartCol = SPREADSHEETS[x].SOURCE.STARTCOL;
var sourceRangeCol = SPREADSHEETS[x].SOURCE.ENDCOL - SPREADSHEETS[x].SOURCE.STARTCOL + 1;
// Get the range of the data you want and the range where you want the data to go
var rowsToCopy = sourceSheet.getLastRow()-sourceStartRow+1; // is +1 too conservative, check...
var rangeToCopy = sourceSheet.getRange(sourceStartRow,sourceStartCol,rowsToCopy, sourceRangeCol);
var dataToCopy = rangeToCopy.getValues();
var numRows = rowsToCopy;
var numColumns = sourceRangeCol;
var intakeStartRow = SPREADSHEETS[x].INTAKE.STARTROW;
var intakeStartCol = SPREADSHEETS[x].INTAKE.STARTCOL;
var rangeToPaste = intakeSheet.getRange(intakeStartRow,intakeStartCol, numRows,numColumns); // WAS FORMERLY 1,20, ..,.. ~DRR 7/14
rangeToPaste.setValues(dataToCopy);
}
}
}

How can I write in Google Sheets my Firebase data?

I have a database in Firebase, and I want to get the data from there and put them in a Google SpreadSheet.
function getData() {
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getSheetByName("Database");
var data = getFirebaseData('contacts');
var [rows, columns] = [sheet.getLastRow(), sheet.getLastColumn()];
var range = sheet.getRange(1,1,1,1);
Logger.log(data)
range.setValue(data)
}
function getFirebaseData(data){
var firebaseUrl = "https://XXXXX.firebaseio.com/";
var secret = 'XXXXXXXX';
var base = FirebaseApp.getDatabaseByUrl(firebaseUrl, secret);
var result = base.getData('contacts');
for(var i in data) {
Logger.log(data[i].eMail + ' ' + data[i].title);
return result;
}
}
and here the image:
No data is shown, and I cannot understand why
Your problem should be solved by completing several steps:
In your getFirebaseData() function, move the return statement outside of the loop;
Instead of looping over data, loop over result (currently, you iterate over each property of the "contacts" String);
Optionally, add checks for getData() returning null or invalid firebaseUrl (in the last case, getData() will cause an error, use try...catch to account for that);
Change base.getData('contacts') to base.getData(data) (isn't it
the reason you pass data to the function?);

Parsing XML Data that I receive from UrlFetch

I want to parse the data I get from UrlFetch into a spreadsheet, but all I'm getting is undefined can someone show me what i'm doing wrong
The xml is at the address https://dl.dropbox.com/u/11787731/Minecraft/bans.xml
function runevery15mins() {
var sheet = SpreadsheetApp.getActiveSpreadsheet().getSheetByName("MC Bans");
sheet.clearContents();
var banURL = "https://dl.dropbox.com/u/11787731/Minecraft/bans.xml";
var banXML = UrlFetchApp.fetch(banURL).getContentText();
var banDOC = Xml.parse(banXML, false);
var mcuser = banDOC.bans;
var x = 0;
for(var c=0; c>mcuser.length;c++){
var name = mcuser.getElement("username")[c].getText();
var date = mcuser.getElement("date")[c].getText();
var reason = mcuser.getElement("reason")[c].getText();
var duration = mcuser.getElement("duration")[c].getText();
}
sheet.appendRow([name, date, reason, duration]);
}
You have some small errors in your code.
For example, the second argument in the for loop needs to be c<mcuser.length.
Using the Xml service documentation, this worked for me
function runevery15mins() {
var sheet = SpreadsheetApp.getActiveSpreadsheet().getSheetByName("MC Bans");
sheet.clearContents();
var banURL = "https://dl.dropbox.com/u/11787731/Minecraft/bans.xml";
var banXML = UrlFetchApp.fetch(banURL).getContentText();
var banDOC = Xml.parse(banXML, false);
// Get all the child nodes from the document element that are 'user' nodes
var mcusers = banDOC.getElement().getElements('user');
for(var c=0; c<mcusers.length;c++){
var user = mcusers[c];
var name = user.getElement('username').getText();
var date = user.getElement('date').getText();
var reason = user.getElement('reason').getText();
var duration = user.getElement('duration').getText();
sheet.appendRow([name, date, reason, duration]);
}
}
Note for example that the sheet.appendRow line is INSIDE the loop, not outside as you had it before. I also deleted the X variable, since I didn't see any purpose for it.
I also created a user variable, which is an XmlElement, to make it easier to understand how to get the different contents of each node.
You were almost there.
Looks like there was another array you needed to drill down into. Also, your call back to the spreadsheet should be in the loop. Try this:
...
var mcuser = banDOC.bans.user;
for(var i in mcuser){
var name = mcuser[i].getElement("username").getText();
var date = mcuser[i].getElement("date").getText();
var reason = mcuser[i].getElement("reason").getText();
var duration = mcuser[i].getElement("duration").getText();
sheet.appendRow([name, date, reason, duration])
}