script slow down with more than 1 user - google-apps-script

Question

I have a sheet with ~18k rows in. I have written a script to divide the number of rows by the number of users who want to get involved in calculating values in the rows in the sheet and then allocating those rows to those users. A collaborative effort to speed things up as each row takes approximately 1s and it takes ~ 6-9 hours to go through it on my own with network issues etc.
When a user opens the sheet, they are allocated a user number and a chunk of rows to work on with 5 users 18k rows breaks down to 3600 rows each.
The issue is that when more than 1 or 2 users are working on the sheet, the performance becomes erratic. Sometimes a row takes 15-20s to be processed. I am saving nothing by having friends share the task.
I read somewhere that only 100 users can work on a sheet at the same time but here I am talking about 5 (up to a maximum of maybe 10 or 11) people working on the sheet, each running maybe 5-8 functions each on a trigger. Right now I have 5 users connected to the sheet, each running 8 functions and the work has slowed almost to a stop.
Does anyone have any experience with this and know of any limits google place on accounts working on a script? Any way to work with apps script to make this work properly?
Thanks for your insights!
Chad
/*NEW CODE WITH TRIGGER - WE START IT UP WITH A CALL TO runTriggersTwoTimes()*/
function runTriggersTwoTimes() {
createSplitWorkTrigger();
ScriptApp.newTrigger('createSplitWorkTrigger')
.timeBased()
.after(80 * 60 * 1000)
.create();
}
function createSplitWorkTrigger() {
ScriptApp.newTrigger('splitWork2')
.timeBased()
.everyMinutes(5)
.create();
ScriptApp.newTrigger('deleteAllTriggers')
.timeBased()
.after(60 * 60 * 1000)
.create();
}
/*This function splits the work of getting the "From" prices between multiple accounts*/
function splitWork2() {
var accountsArray = [user1, user2, user3, user4, user5#gmail.com];
var numberAccounts = accountsArray.length;
var spreadsheet = SpreadsheetApp.getActiveSpreadsheet();
var sheet = spreadsheet.getSheetByName("CK Formatted");
var RANGE = sheet.getDataRange();
var rangeVals = RANGE.getValues();
var numberRows = rangeVals.length;
var totalChunkSize = Math.floor(numberRows / numberAccounts);
var userName = Session.getActiveUser().getEmail();
var userNumber;
for (i = 0; i < numberAccounts; i++) {
if (userName == accountsArray[i]) {
userNumber = i;
}
}
var usersStartRow = userNumber * totalChunkSize + 1
var usersLastRow = usersStartRow + totalChunkSize - 1;
if (userName == accountsArray[numberAccounts - 1]) {
usersLastRow = numberRows;
}
// This one does the main work
findFromPricesByChunks2(userNumber, usersStartRow, usersLastRow, 110, totalChunkSize);
}
/*This function adds the "From" prices in chunks*/
function findFromPricesByChunks2(userNumber, startRow, lastRow, chunkSize, totalChunkSize, checkingRound = false) {
if (startRow >= lastRow) {
var allTriggers = ScriptApp.getProjectTriggers();
for (var i = 0; i < allTriggers.length; i++) {
ScriptApp.deleteTrigger(allTriggers[i]);
}
return;
}
var spreadsheet = SpreadsheetApp.getActiveSpreadsheet();
var sheet = spreadsheet.getSheetByName("CK Formatted");
var RANGE = sheet.getDataRange();
var rangeVals = RANGE.getValues();
Logger.log(rangeVals.length);
var addedPrices = 0;
for (var i = startRow; i <= lastRow; i++) {
if (addedPrices == chunkSize) {
return;
}
var cellValue = sheet.getRange(`A${i + 1}`).getValue();
Logger.log('Cell Value:' + cellValue);
if (cellValue == '') {
Utilities.sleep(1000);
var html = UrlFetchApp.fetch(URL).getContentText();
var $ = Cheerio.load(html);
var s = $('#table .price-container .font-weight-bold').first().text();
// if we fail to get a value because of network issues or whatever this next piece is going ahead and putting a full stop in what should be an empty box. WIll think about this
if (s) {
s = s.replace(".", ",");
s = s.substring(0, s.lastIndexOf(',')) + '.' + s.substring(s.lastIndexOf(',') + 1);
}
fromArrayCell = sheet.getRange(`A${i + 1}`);
fromArrayCell.setValue(s);
addedPrices++;
}
}
var sheetCk = spreadsheet.getSheetByName("CK");
// This adds the amount of processed items to the processed rows counter (because now there are multiple workers working
// at the same time)
if (!checkingRound) {
sheetCk.getRange(userNumber + 2, 9).setValue(sheetCk.getRange(userNumber + 2, 9).getValue() + addedPrices);
} else {
sheetCk.getRange(userNumber + 2, 10).setValue(sheetCk.getRange(userNumber + 2, 10).getValue() + addedPrices);
}
}
function deleteAllTriggers() {
var allTriggers = ScriptApp.getProjectTriggers();
for (var i = 0; i < allTriggers.length; i++) {
ScriptApp.deleteTrigger(allTriggers[i]);
}
}

Answer 1

The single most important thing to come out of this post was the limit of 30 executions which I was told before I posted any code :p I didn't know about that and it was the crux of the problem I was having. Bear in mind that the original code (a few versions back) was trying to cram 13 users in with 8 processes each :)
After drifting for a few days I had a good sleep and came up with this new improved version of the above:
function splitWork() {
var spreadsheet = SpreadsheetApp.getActiveSpreadsheet();
var sheetCk = spreadsheet.getSheetByName("CK");
var sheet = spreadsheet.getSheetByName("CK Formatted");
var RANGE = sheet.getDataRange();
var rangeVals = RANGE.getValues();
var userName = Session.getActiveUser().getEmail();
Logger.log(userName)
var accountsArray = [gmail1, gmail2, gmail3, gmail4, gmail5];
var numberRows = rangeVals.length;
var numberAccounts = accountsArray.length;
var totalChunkSize = Math.floor(numberRows / numberAccounts);
var userNumber;
for (i = 0; i < numberAccounts; i++) {
if (userName == accountsArray[i]) {
userNumber = i;
}
}
var usersStartRow = (userNumber * totalChunkSize) + 1; // = 1
var usersLastRow = usersStartRow + totalChunkSize - 1; // = 211
if(userName == accountsArray[numberAccounts - 1]) { usersLastRow = numberRows; }
var completeRowsOriginal = sheetCk.getRange(userNumber + 2, 9).getValue() // = 109
usersStartRow = usersStartRow + completeRowsOriginal // = 110
for (var i = usersStartRow; i <= usersLastRow; i++) { // 110 - 211
var cellValue = sheet.getRange(`A${i + 1}`).getValue();
var completeRows = 0
if (cellValue == '') {
Utilities.sleep(1000);
var html = UrlFetchApp.fetch(URL).getContentText();
var $ = Cheerio.load(html);
var fromArrayCell = sheet.getRange(`A${i + 1}`);
if(sheetCk.getRange(userNumber + 2, 10).getValue() != 1){
var s = $('#table .price-container .font-weight-bold').first().text();
}
else if(sheetCk.getRange(userNumber + 2, 10).getValue() == 1) {
var s = $('#table .price-container .font-weight-bold').first().text();
if(!s) {
var s = $('#tabContent-info .info-list-container dd:nth-child(12)').text();
if(s) { fromArrayCell.setBackground('#ff8680'); }
}
}
if (s) {
s = s.replace(".", ",");
s = s.substring(0, s.lastIndexOf(',')) + '.' + s.substring(s.lastIndexOf(',') + 1);
}
fromArrayCell.setValue(s);
completeRows++
sheetCk.getRange(userNumber + 2, 9).setValue(sheetCk.getRange(userNumber + 2, 9).getValue() + 1)
}
if(completeRows >= 100 || usersStartRow + completeRows >= usersLastRow) {break}
}
if(sheetCk.getRange(userNumber + 2, 9).getValue() + usersStartRow >= usersLastRow) {
if(sheetCk.getRange(userNumber + 2, 10).getValue() != 1) {
sheetCk.getRange(userNumber + 2, 9).setValue(0)
sheetCk.getRange(userNumber + 2, 10).setValue(1)
}
}
else { deleteAllTriggers() }
}
Which works like a charm.
A user opens the sheet, is allocated a chunk in the background, presses a button to start a trigger to work on their chunk every 5 minutes and if they are feeling frisky they can run the function above while waiting for the trigger to start (also from a button). They'll scrape values for 100 rows then wait for another trigger. After they have gone through their whole chunk their counter resets and their master counter is set to 1. They then make a second pass over their chunk filling in any gaps due to network errors. If they can't get a value from the regular place on the second pass they can get a value from another place but it is marked with a red background as it is not reliable.
Don't think there is anything else left to say about this one. Try it yourselves if you have a large sheet of data to churn through and want to do it in less time with help from your friends with google accounts.
PS: CK is the sheet which is used to hold counters only
PPS!! If you literally do it as above, you will manage to process about 1 row per second. The real power coms from getting all the helper account to make their own copy of the main sheet and work on that. Here we'd get 13 rows per second processed, for instance. Then it's a simple matter to recombine everything into single sheet when they're done.