make variable from sheet tabs and one template - google-apps-script

function createDocs() {
var headers = Sheets.Spreadsheets.Values.get('1wjmfG-mTYFw_JQZIondppy-VxffdV3fFzcEi8Og', 'Juli 2021!A5:K5');
var tactics = Sheets.Spreadsheets.Values.get('1wjmfG-mTYFw_JQZIondppy-VoCguU_VgVfdsdcEi8Og', 'Juli 2021!H6:K4000');
var templateId = '1B2IzVd03MmoPtT7Tcvce8I9wXGrb6fYXCnz8JikAs';
for(var i = 0; i < tactics.values.length; i++){
var textId = tactics.values[i][0];
var titel = tactics.values[i][1];
var words = tactics.values[i][3];
//Make a copy of the template file
var documentId = DriveApp.getFileById(templateId).makeCopy().getId();
//Rename the copied file
DriveApp.getFileById(documentId).setName('NEW ' + textId +' '+ titel + ' ' + words);
//Get the document body as a variable
var body = DocumentApp.openById(documentId).getBody();
//Insert the textid name
body.replaceText('##textID##', textId)
//Insert the titel name
body.replaceText('##subject##', titel)
//Insert the words name
body.replaceText('##words##', words)
//Append tactics
parseTactics(headers.values[0], tactics.values[i], body);
}
}
function parseTactics(headers, tactics, body){
for(var i = 1; i < tactics.length; i++){
{tactics[i] != '' &&
body.appendListItem(headers[i] + ' | ' + tactics[i] + ' net').setGlyphType(DocumentApp.GlyphType.BULLET);
}
}
}
So I have this code that works fine on specific tabs and creating docs based on template.
How can i make it so I the code works for all tabs?
All tabs of interest have the format: "month year".
There are some tabs not of interest that have different format.
All tabs have there own corresponding folders.
Goal is to just have a template file I can copy the each folder that needs docs created.

If your sheet/tabs are something like this: November 2021
Then you could use something like this to filter them from the rest of your sheets:
function selectSheets() {
const mA = [...Array.from(new Array(12).keys(), x => Utilities.formatDate(new Date(2021, x), Session.getScriptTimeZone(), "MMMM"))];
const ss = SpreadsheetApp.getActive();
let shts = ss.getSheets().filter(sh => {
let t = sh.getName().split(' ');
if (~mA.indexOf(t[0]) && t[1].match(/\d{4}/)) {
return true;
} else {
return false;
}
}).map(sh => sh.getName());
Logger.log(shts.join(','))
}

Related

Method template.copy is not working - Google Script

I am facing a problem that is leaving me without a real solution.
I want that the script create a new Google Doc copy from a template, then transfer data from Sheet toward the new Doc.
The script is running without a bug, but fail to create a copy of the document as it supposed to do.
function transferData() {
var sheet = SpreadsheetApp.getActive().getSheetByName("Tour de ContrĂ´le");
var templateId = "119ORijiWNQEdpG4KqjNCYzilbf_ob7SMdOPXUK3woMQ";
var folder = DriveApp.getFolderById("166g1akWuHR3MtoDzbh9ydMhEpr4euCts");
var data = sheet.getDataRange().getValues();
for (var i = 3; i < data.length; i++) {
if (data[i][12] === true) {
var nom = data[i][3];
var nFonction = data[i][6];
var template = DocumentApp.openById(templateId);
var newDoc = template.copy("Mouvement_" + nFonction + "_" + nom + "_EMBA");
newDoc.getAs(MimeType.GOOGLE_DOCS).getFile().moveTo(folder);
var docBody = newDoc.getBody();
var matricule = data[i][4];
var compta = data[i][16];
var division = data[i][1];
var fonction = data[i][6];
var respDoss = sheet.getRange("GDD!B3").getValue();
var endosGDD = sheet.getRange("GDD!B6").getValue();
var sign = sheet.getRange("GDD!B7").getValue();
docBody.replaceText("{Nom}", nom);
docBody.replaceText("{Matricule}", matricule);
docBody.replaceText("{NFonction}", nFonction);
docBody.replaceText("{Compta}", compta);
docBody.replaceText("{division}", division);
docBody.replaceText("{Fonction}", fonction);
docBody.replaceText("{RespDoss}", respDoss);
docBody.replaceText("{EndosGDD}", endosGDD);
docBody.replaceText("{Sign}", sign);
}
}
}
There's no DocumentApp.openById(templateId).copy() method. To make a copy of a doc you should use Driveapp instead.
You should change these lines of code
var template = DocumentApp.openById(templateId);
var newDoc = template.copy("Mouvement_" + nFonction + "_" + nom + "_EMBA");
newDoc.getAs(MimeType.GOOGLE_DOCS).getFile().moveTo(folder);
For these ones:
var template = DriveApp.openById(templateId);
var newDoc = template.makeCopy("Mouvement_" + nFonction + "_" + nom + "_EMBA", folder);
makeCopy(name, destination)
With SpreadsheetApp you can make a copy of a spreadsheet but you can't choose the folder where it will be created (it's created in the same folder where your spreadsheet "file" is located).
copy(name)
Finally, I modified const googleDocTemplate = DriveApp.getFolderById by const googleDocTemplate = DriveApp.getFileById
It worked.
Thanks for your help.
function transferData() {
//Here we store the sheet as a variable
const sheet = SpreadsheetApp.getActiveSpreadsheet().getSheetByName("Tour de ContrĂ´le");
//This value should be the id of your document template that we created in the last step
const googleDocTemplate = DriveApp.getFileById('119ORijiWNQEdpG4KqjNCYzilbf_ob7SMdOPXUK3woMQ');
//This value should be the id of the folder where you want your completed documents stored
const destinationFolder = DriveApp.getFolderById('166g1akWuHR3MtoDzbh9ydMhEpr4euCts');
const respDoss = sheet.getRange("GDD!B3").getValue();
const endosGDD = sheet.getRange("GDD!B6").getValue();
const sign = sheet.getRange("GDD!B7").getValue();
const rows = sheet.getDataRange().getValues();
//Logger.log(rows)
rows.forEach(function(row, index){
//Here we check if this row is the headers, if so we skip it
if (index === 0) return;
//Here we check if a document has already been generated by looking at 'Generated date', if so we skip it
//if (row[16] === true);
//Here we check if the value in column Q (index 17) is true, if not we skip this row
if (index >= 4) {
if (row[17] !== true){
return;
}
}
Logger.log(rows);
//Using the row data in a template literal, we make a copy of our template document in our destinationFolder
const copy = googleDocTemplate.makeCopy(`Mouvement_${row[6]}, ${row[3]}_EMBA` , destinationFolder);
//Once we have the copy, we then open it using the DocumentApp
const doc = DocumentApp.openById(copy.getId());
//All of the content lives in the body, so we get that for editing
const body = doc.getBody();
...

Attachments: undefined Apps Script

I am trying to pull an attachment from an email which is a zip file and send that file to Google Analytics to upload the data. I am struggling with the getAttachment part in the script as it is showing as undefined. I have no idea what I am doing wrong. Any help would be greatly appreciated. I have now fixed where it is pulling the right attachment by using: var attachments = messages[0].getAttachments(); However I am now getting a new error where it seems to be looking for a title of the attachment but it is still showing as undefined.
function refundImport() {
/// use custom report to schedule the email - will need to adjust the processCsv() function based on your schema
/// if you are using a non-bing data source - you will probably need to adjust the findCsvAttachments() function as it grabs a zip file now
var CONFIG = {
'emailSubject': 'Refunded or Partially Refunded Orders - TKS',
'customDataSourceId': 'xxxxxxxxxxxxxxxxxxxxx',
'now': new Date(),
'zipFileName': 'refunded_or_partially_refunded_orders.zip',
//'csvFileName': 'refunded_or_partially_refunded_orders.csv',
'analyticsAccountId': '12345678',
'analyticsPropertyId': 'UA-12345678-1'
}
//adds one whole day to a date object - can take negative days if you want yesterday etc
function addDaysToDate(DATE, DAYS) {
var newDate = DATE.getTime() + DAYS * 3600000 * 24;
var newDate1 = new Date(newDate);
return newDate1;
}
//takes a date object and formats it as a string
function formatDateAsString(DATE) {
var dateString = Utilities.formatDate(DATE, 'GMT+12:00', 'yyyy/MM/dd');
return dateString;
}
//after and before must be date strings - use the above function
function grabEmailAttachments(SUBJECT, AFTER, BEFORE) {
var query = 'subject:' + SUBJECT + ' ' + 'has:attachment after:' + AFTER + ' ' + 'before:' + BEFORE;
//assumes only 1 will match - if more than 1 - will match the first one
var thread = GmailApp.search('in:inbox from:"noreply#highviewapps.com"');
var messages = thread[0].getMessages();
var content = messages[0].getPlainBody();
//var attachments = thread.getAttachments()[0];
var attachments = messages[0].getAttachments();
//thread.moveToTrash();
return attachments;
}
// finds csv attachment and creates 2d array of row,column e.g. csv[0][1] = value in row 0 column 1 of csv
function findCsvAttachment(attachments, zipFileToSearch, fileNameToSearch) {
var counter = 0;
for (i = 0; i < attachments.length; i++) {
if (attachments[i].getName().search(zipFileToSearch) != -1) {
var unzip = Utilities.unzip(attachments[i]);
var csvData = Utilities.parseCsv(unzip[0].getDataAsString(), ",");
counter = counter + 1;
}
}
if (counter == 0) {
Logger.log('No file with ' + fileNameToSearch + ' in its name was found.');
}
if (counter == 1) {
return csvData;
}
if (counter > 1) {
Logger.log('More than 1 file with ' + fileNameToSearch + ' in its name was found - the last one was used.');
}
}
function processCsv(csvData, date) {
var headers = 'ga:transactionId,ga:productSku,ga:productPrice,ga:quantityRefunded,ga:transactionRevenue';
var dataForUpload = headers;
return dataForUpload;
}
//assumes media dataType for upload https://developers.google.com/analytics/devguides/config/mgmt/v3/mgmtReference/management/uploads/uploadData
function uploadDataToAnalytics(data, accountId, webPropertyId, customDataSourceId) {
var dataBlob = Utilities.newBlob(data, "application/octet-stream");
var upload = Analytics.Management.Uploads.uploadData(accountId, webPropertyId, customDataSourceId, dataBlob);
return upload;
}
///////// ACTUAL IMPLEMENTATION OF SCRIPT /////////////
var tomorrowString = formatDateAsString(addDaysToDate(CONFIG.now, 1));
var yesterdayString = formatDateAsString(addDaysToDate(CONFIG.now, -1));
var todayString = formatDateAsString(CONFIG.now);
var emailAttachments = grabEmailAttachments(CONFIG.emailSubject, yesterdayString, tomorrowString);
var csv = findCsvAttachment(emailAttachments, CONFIG.zipFileName, CONFIG.csvFileName);
var csvForUpload = processCsv(csv, yesterdayString);
var analyticsUpload = uploadDataToAnalytics(csvForUpload, CONFIG.analyticsAccountId, CONFIG.analyticsPropertyId, CONFIG.customDataSourceId);
}
Try changing these lines of code
From:
var attachments = messages.getAttachments();
and
var unzip = Utilities.unzip(attachments[i]);
To:
var attachments = messages[0].getAttachments();
and
var unzip = Utilities.unzip(attachments[i].copyBlob());

Add another row content to the .txt

I just got to work the script to export the content of a row to a .txt and put a =, but now i would like to know how i can add the content of other row to the .txt like i will show on the screenshot.
This is the code that i currently have but i dont properly make to work.
function exporttotxt() {
var ss = SpreadsheetApp.getActive();
var sheet = ss.getActiveSheet();
var range = sheet.getRange('G3:G' + sheet.getLastRow());
var rows = range.getValues().filter(([g]) => g.toString() != "");
var range2 = sheet.getRange('I3:I' + sheet.getLastRow());
var rows2 = range2.getValues().filter(([i]) => i.toString() != "");
var fileName="exported.txt";
var folderName="Videos";
var data = rows.splice(0);
var data2 = rows2.splice(0);
var str = data.map(function(e) {return e.join()}).join("=") + "=" + data2.map(function(e) {return e.join()}).join("\n");
var content = str;
// get list of folders with matching name
var folderList = DriveApp.getFoldersByName(folderName);
if (folderList.hasNext()) {
// found matching folder
var folder = folderList.next();
// search for files with matching name
var fileList = folder.getFilesByName(fileName);
if (fileList.hasNext()) {
// found matching file - append text
var file = fileList.next();
var combinedContent = content;
file.setContent(combinedContent);
}
else {
// file not found - create new
folder.createFile(fileName, content);
}
}
}
I believe your goal as follows.
You want to create a text by merging the values of colums "G" and "I".
In your situation, how about retrieving the values from the columns "G" and "I" by one getValues? By this, the process cost will be lower and the script becomes a bit simple. When this is reflected to your script, it becomes as follows.
From:
var range = sheet.getRange('G3:G' + sheet.getLastRow());
var rows = range.getValues().filter(([g]) => g.toString() != "");
var range2 = sheet.getRange('I3:I' + sheet.getLastRow());
var rows2 = range2.getValues().filter(([i]) => i.toString() != "");
var fileName="exported.txt";
var folderName="Videos";
var data = rows.splice(0);
var data2 = rows2.splice(0);
var str = data.map(function(e) {return e.join()}).join("=") + "=" + data2.map(function(e) {return e.join()}).join("\n");
var content = str;
To:
var range = sheet.getRange('G3:I' + sheet.getLastRow());
var rows = range.getValues().filter(([g, _, i]) => g.toString() != "" && i.toString() != "");
var fileName="exported.txt";
var folderName="Videos";
var data = rows.splice(0);
var str = data.map(([g, _, i]) => `${g}=${i}`).join("\n");
var content = str;

Google Sheets Script: email with a HTML Table that only has a set of

In Google Spreadsheet I want to be able to send a email with a HTML Table within the Email. I have accomplished this below and would like to now extend the functionality more. Now I am trying to understand / find a way on how to filter the getRange based on a variable.
Example:
If Column C = Todays Date I want to return all rows with Todays date into the HTML Table in the Email. I have been playing with the GetRange but when adjusting it something other then a Range it breaks. How should I go about doing this?
function sendEmail() {
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getActiveSheet();
var range = sheet.getDataRange();
var recipient = 'email#gmail.com'
var subject = 'Subject'
var date = Utilities.formatDate(new Date(), "GMT+1", "dd/MM/yyyy")
var schedRange = sheet.getRange("A1:L21"); // Trying to understand
//var schedRange = sheet.getRange(Col == 3 && compare == date);
// Put Name & Date into email first.
// We only want the schedule within borders, so
// these are handled separately.
var body = '<div style="text-align:center;display: inline-block;font-family: arial,sans,sans-serif">'
body += '<H1>'+ 'Deployment Table Header ' +'</H1>';
body += '<H2>'
body += getHtmlTable(schedRange);
body += '</div>';
debugger;
recipient = 'email#gmail.com'; // For debugging, send only to self
GmailApp.sendEmail(recipient, subject, "Requires HTML", {htmlBody:body})
}
/**
* Return a string containing an HTML table representation
* of the given range, preserving style settings.
*/
function getHtmlTable(range){
var ss = range.getSheet().getParent();
var sheet = range.getSheet();
startRow = range.getRow();
startCol = range.getColumn();
lastRow = range.getLastRow();
lastCol = range.getLastColumn();
// Read table contents
var data = range.getValues();
// Get css style attributes from range
var fontColors = range.getFontColors();
var backgrounds = range.getBackgrounds();
var fontFamilies = range.getFontFamilies();
var fontSizes = range.getFontSizes();
var fontLines = range.getFontLines();
var fontWeights = range.getFontWeights();
var horizontalAlignments = range.getHorizontalAlignments();
var verticalAlignments = range.getVerticalAlignments();
// Get column widths in pixels
var colWidths = [];
for (var col=startCol; col<=lastCol; col++) {
colWidths.push(sheet.getColumnWidth(col));
}
// Get Row heights in pixels
var rowHeights = [];
for (var row=startRow; row<=lastRow; row++) {
rowHeights.push(sheet.getRowHeight(row));
}
// Build HTML Table, with inline styling for each cell
var tableFormat = 'style="font-size: 10px; border:1px solid black;border-collapse:collapse;text-align:center" border = 1 cellpadding = 1';
var html = ['<table '+tableFormat+'>'];
// Column widths appear outside of table rows
for (col=0;col<colWidths.length;col++) {
html.push('<col width="'+colWidths[col]+'">')
}
// Populate rows
for (row=0;row<data.length;row++) {
html.push('<tr height="'+rowHeights[row]+'">');
for (col=0;col<data[row].length;col++) {
// Get formatted data
var cellText = data[row][col];
if (cellText instanceof Date) {
cellText = Utilities.formatDate(
cellText,
ss.getSpreadsheetTimeZone(),
'M/d');
}
var style = 'style="'
+ 'color: ' + fontColors[row][col]+'; '
+ 'font-family: ' + fontFamilies[row][col]+'; '
+ 'font-size: ' + fontSizes[row][col]+'; '
+ 'font-weight: ' + fontWeights[row][col]+'; '
+ 'background-color: ' + backgrounds[row][col]+'; '
+ 'text-align: ' + horizontalAlignments[row][col]+'; '
+ 'vertical-align: ' + verticalAlignments[row][col]+'; '
+'"';
html.push('<td ' + style + '>'
+cellText
+'</td>');
}
html.push('</tr>');
}
html.push('</table>');
return html.join('');
}
As stated in the comments, you can filter out any rows you don't want when you build your HTML table:
function getHtmlTable(range){
// ... your code...
var today = new Date();
for (row=0;row<data.length;row++) {
var row_date = data[row][2]; // Assuming date is in 3rd column
if(sameDay(new Date(row_date), today){ // See note
html.push('<tr height="'+rowHeights[row]+'">');
for (col=0;col<data[row].length;col++) {
// ... your code ...
}
}
}
}
function sameDay(d1, d2) {
return d1.getFullYear() === d2.getFullYear() &&
d1.getMonth() === d2.getMonth() &&
d1.getDate() === d2.getDate();
}
A few notes:
Calling new Date(row_date) will only work if the date is formatted correctly for the Date constructor. If it is not, you need to parse the date before creating the Date object.
sameDay function borrowed from this answer.

compare two spreadsheet and output the difference using google app scripts

well, i'm trying to do what described in title. Both spreadsheets have only one sheet that are the ones i'm comparing. One spreadsheet is and update of the other, so i'm trying to get only new content. (if it were a fc (dos command) like function this would be easy...)
After doing some search, i have the folloing script that should work on most cases, that uses arrays for each sheet.
function test() {
var Folder = DriveApp.getFoldersByName('theFolder').next();
var FolderId =Folder.getId();
//call old_spreadsheet
var searchFor ="fullText contains 'sheet_old' and '" + FolderId + "' in parents";
var files = DriveApp.searchFiles(searchFor);
var old_file = files.next();
var old_spreadsheet = SpreadsheetApp.openById(old_file.getId());
var old_sheet = old_spreadsheet.getSheets()[0];
var old_sheetname = old_sheet.getName();
var old_array = old_sheet.getDataRange().getValues();
Logger.log(old_file.getName() + ' : ' + old_sheetname + ' : ' + old_array.length);
//call spreadsheet
var searchFor ="fullText contains 'sheet' and '" + FolderId + "' in parents";
var files = DriveApp.searchFiles(searchFor);
var file = files.next();
var spreadsheet = SpreadsheetApp.openById(file.getId());
var sheet = spreadsheet.getSheets()[0];
var sheetname = sheet.getName();
var array = sheet.getDataRange().getValues();
Logger.log(file.getName() + ' : ' + sheetname + ' : ' + array.length);
var newarray = getNewData(array,old_array);
Logger.log('there are ' + newarray.length + 'different rows');
}
function getNewData(array1,array2){
var diff =array2;
for (var i = 0; i<array1.length; i++){
var duplicate = false;
for (var j = 0;j<diff.length;j++){
if (array1[i].join() == diff[j].join()){
Logger.log('duplicated line found on rows ' + i + ':' + j);
diff.splice(j,1);
var duplicate= true;
break;
}
}
if (duplicate==false) {
Logger.log('not duplicated line found on row ' + i);
diff.push(array1[i]);
}
}
return diff;
}
The thing is that the files are too big, almost 30000 rows, so the scripts exceed 5 minutes limit for execution.
Is there a way to improve this, like for instance, eliminate the inner for loop?
Or there is a way to do it in parts? like first the first 5000 rows, and so on.
Regards,
EDIT: after analizing the spreadsheet a little, i found out that there is a ID for every row, so now i can concentrate the search only in one column of each spreadsheet. So here is my new implementation:
function test(){
var Folder = DriveApp.getFoldersByName('theFolder').next();
var FolderId =Folder.getId();
//call old_spreadsheet
var searchFor ="fullText contains 'sheet_old' and '" + FolderId + "' in parents";
var files = DriveApp.searchFiles(searchFor);
var old_file = files.next();
var old_spreadsheet = SpreadsheetApp.openById(old_file.getId());
var old_sheet = old_spreadsheet.getSheets()[0];
var old_sheetname = old_sheet.getName();
var old_array = old_sheet.getDataRange().getValues();
Logger.log(old_file.getName() + ' : ' + old_sheetname + ' : ' + old_array.length);
//call spreadsheet
var searchFor ="fullText contains 'sheet' and '" + FolderId + "' in parents";
var files = DriveApp.searchFiles(searchFor);
var file = files.next();
var spreadsheet = SpreadsheetApp.openById(file.getId());
var sheet = spreadsheet.getSheets()[0];
var sheetname = sheet.getName();
var array = sheet.getDataRange().getValues();
Logger.log(file.getName() + ' : ' + sheetname + ' : ' + array.length);
//The COlumn has an indicator, so i search for that. I don't control the formatting of the files, so i search in both spreadsheet for the indicator
var searchString = 'NAME';
for (var i = 0; i < old_array.length; i++) {
for (var j = 0; j < old_array[i].length; j++) {
if (old_array[i][j] == searchString) {
var Row_old = i+1;
var Column_old = j;
break;
}
}
if (Row_old != undefined){
break;
}
}
for (var i = 0; i < array.length; i++) {
for (var j = 0; j < array[i].length; j++) {
if (array[i][j] == searchString) {
var Row = i+1;
var Column = j;
break;
}
}
if (Row != undefined){
break;
}
}
Logger.log(Row_old+':::'+Column_old+'\n'+Row+':::'+Column);
var diff_index =[];
var row_ind = 0;
for (var i=Row;i<array.length;i++){
Logger.log(i);
var existe = ArrayLib.indexOf(old_array, Column_old, array[i][Column]);
if (existe==-1){
Logger.log(row_ind+'!!!');
diff_index[row_ind]=i;
row_ind++;
}
}
Logger.log(diff_index);
}
This still run out of time... I will now try to incorporate your comments.
Your script has a few major bottlenecks that slow it down massively:
Starting both loops at 0 every time makes its runtime explode
splicing every time you find a duplicate requires to move the array around
string concatenating an array on every iteration
We can circumvent these issues by:
sorting the second range once
I'm sure there's something clever to be done by iteratively binary searching through every column but we'd have to resort every time so we'll binary search the first column and then do a linear search.
We will use ArrayLib for the sorting (I hope it's a fast sorting algorithm).
Let's start with a function to find the first row where the first column matches a value (the first column of the current row):
function firstRowMatchingCol1(target, lookupRange) {
var min = 0;
var max = lookupRange.length - 1;
var guess;
var guessVal;
while(min <= max) {
guess = (min + max) / 2 | 0;
guessVal = lookupRange[guess][0];
if (guessVal < target) {
min = guess + 1;
} else if (guessVal > target) {
max = guess - 1;
} else {
while (guess > 0 && lookupRange[guess - 1][0] === target) {
guess -= 1;
}
return guess;
}
}
return -1;
}
Now we can go linearly go through every row and check if the columns match until the first column doesn't match anymore.
function matchExists(row, lookupRange) {
var index = firstRowMatchingCol1(row[0], lookupRange);
if (index === -1) {return false;}
while (index < lookupRange.length && lookupRange[index][0] === row[0]) {
for (var col = 1; col < row.length; col++) {
if (row[col] !== lookupRange[index][col]) {break;}
if (col === row.length - 1) {return true;} // This only works if the ranges are at least two columns wide but if they are one column wide you can just check if index > -1
}
index += 1;
}
return false;
}
And finally we can get the duplicates like this:
function getNonDuplicates(r1, r2) {
r2 = ArrayLib.sort(r2, 0, true);
return r1.filter(function(row) {return !matchExists(row, r2);});
}
Like mTorres' code this is untested
The solution I'm proposing is a "hack" around the time limit. But if you want a cleaner solution, you could, if possible, reorganize and make your code more efficient by having the arrays ordered somehow.
You don't specify the data inside array1 and array2, if rows had some sort of ID field you could order by this ID and check row i on array1 and row i on array2 instead of comparing every row in array1 with every row in array2 (which is extremely inefficient with 30000 rows).
If your data does not have an ID field to order the rows, then what you could is something based on my proposed solution: add a track for every compared row on array1. When the run reaches the time limit then you run again the function but starting from the last compared row (you would know which was because you'll be tracking the compared rows), and when the second run times out you repeat, and so on.
Every time you run your comparison you ask if it's the first run (or use a boolean - I prefer to ask the user, this way you won't forget to change the boolean), if it's the first run, you delete the tracking
column, if it's not the first run, you'll start with the next to last tracked row so basically continuing your script where it ended. I've been using this technique with good results.
In code (untested, so check it out before running it with real data):
/**
* Only checks if it's the first run and calls the real work function
*/
function test() {
var firstRun = "yes" === Browser.msgBox("Question", "Is this the first run?", Browser.Buttons.YES_NO);
doTest(firstRun);
}
/**
* Gets the data of the 2 spreadsheets and also the starting
* row
*/
function doTest(firstRun) {
var Folder = DriveApp.getFoldersByName('theFolder').next();
var FolderId = Folder.getId();
//call old_spreadsheet
var searchFor ="fullText contains 'sheet_old' and '" + FolderId + "' in parents";
var files = DriveApp.searchFiles(searchFor);
var old_file = files.next();
var old_spreadsheet = SpreadsheetApp.openById(old_file.getId());
var old_sheet = old_spreadsheet.getSheets()[0];
var old_sheetname = old_sheet.getName();
var old_array = old_sheet.getDataRange().getValues();
/**
* Here is the code to create the tracking hability
*/
var strartFromRow = 0; // 0 because row 1 is array 0 index when you getValues();
var trackSheet = old_spreadsheet.getSheetByName("Tracking");
if (trackSheet === null) {
trackSheet = old_spreadsheet.insertSheet("Tracking");
}
if (firstRun) {
trackSheet.getRange("A:A").clearContent(); // make sure there no row is tracked yet
}
else {
// we have to continue from the previous row, keep in mind you're making the comparison
// with array which is 0 based, but sheet is 1 based, but you want the next one so getLasRow()
// should be the first item to compare on your array
strartFromRow = trackSheet.getLastRow();
}
Logger.log(old_file.getName() + ' : ' + old_sheetname + ' : ' + old_array.length);
//call spreadsheet
var searchFor ="fullText contains 'sheet' and '" + FolderId + "' in parents";
var files = DriveApp.searchFiles(searchFor);
var file = files.next();
var spreadsheet = SpreadsheetApp.openById(file.getId());
var sheet = spreadsheet.getSheets()[0];
var sheetname = sheet.getName();
var array = sheet.getDataRange().getValues();
Logger.log(file.getName() + ' : ' + sheetname + ' : ' + array.length);
// when you call the DIFF function, pass the tracking sheet and the start Row
var newarray = getNewData(array,old_array, trackSheet, startFromRow);
Logger.log('there are ' + newarray.length + 'different rows');
}
/**
* Creates a diff array using array1 and array2
* It marks each element on array1 once it has checked if it's in array2
*/
function getNewData(array1, array2, trackingSheet, startFromRow){
var logRow = trackingSheet.getLastRow();
var diff = array2;
for (var i = startFromRow; i < array1.length; i++){
var duplicate = false;
for (var j = 0; j < diff.length;j++){
if (array1[i].join() == diff[j].join()){
Logger.log('duplicated line found on rows ' + i + ':' + j);
diff.splice(j,1);
duplicate = true;
break;
}
}
if (duplicate === false) {
Logger.log('not duplicated line found on row ' + i);
diff.push(array1[i]);
}
trackingSheet.getRange(logRow++, 1).setValue("Checked!"); // Mark i row as checked
}
return diff;
}
Here's an alternate solution that gets around the time limit. Create a new dedicated spreadsheet along with a custom sidebar. The sidebar will require you to create some HTML that will ultimately be embedded and rendered in an iframe on the client. You can embed pure javascript into the HTML via script tags.
The beauty of this approach is that these scripts will not run server-side but on the client independently of Google Apps Script's server-side environment and are not subject to the 6 minute limit. Moreover, they can also call functions in your Google Script. So one approach would be to have the client-side scripts call a Google Script function to retrieve the requisite data, do all the heavy processing in the client-side scripts, and then send the results back to the server-side script to update the sheet.
Here's a link to setting up a custom sidebar to get you started:
https://developers.google.com/apps-script/guides/dialogs#custom_sidebars
Finally, i decided to go for the Cache service option, here is the code and i'm testing it to see if i keep with this.
function getNewData() {
//deleting triggers
var triggers = ScriptApp.getProjectTriggers();
for (var i = 0; i < triggers.length; i++) {
if (triggers[i].getHandlerFunction()=='getNewData'){
ScriptApp.deleteTrigger(triggers[i]);
}
}
//max running time = 5.5 min
var MAX_RUNNING_TIME = 330000;
var startTime= (new Date()).getTime();
//get cache
var cache = CacheService.getUserCache();
var downloaded =JSON.parse(cache.get('downloaded'));
var compared =JSON.parse(cache.get('compared'));
//start
if (downloaded==1 && compared!=1){
//folder
var Folder = DriveApp.getFoldersByName('theFolder').next();
var FolderId = licitacionesFolder.getId();
//call old_spreadsheet
var searchFor ="fullText contains 'sheet_old' and '" + FolderId + "' in parents";
var files = DriveApp.searchFiles(searchFor);
var old_file = files.next();
var old_spreadsheet = SpreadsheetApp.openById(old_file.getId());
var old_sheet = old_spreadsheet.getSheets()[0];
var old_array = old_sheet.getDataRange().getValues();
//call spreadsheet
var searchFor ="fullText contains 'sheet' and '" + FolderId + "' in parents";
var files = DriveApp.searchFiles(searchFor);
var file = files.next();
var spreadsheet = SpreadsheetApp.openById(old_file.getId());
var sheet = spreadsheet.getSheets()[0];
var array = sheet.getDataRange().getValues();
Logger.log(array.length+'::'+old_array.length);
// Column
var searchString = 'NAME';
var RC = getColumn(array,searchString);
var Row = RC.Row;
var Column = RC.Column;
var RC = getColumn(old_array,searchString);
var Row_old = RC.Row;
var Column_old = RC.Column;
Logger.log(Row_old+':::'+Column_old+'\n'+Row+':::'+Column);
//compare
var diff_index =JSON.parse(cache.get('diff_index'));
var row_ind =JSON.parse(cache.get('row_ind'));
var Roww =JSON.parse(cache.get('Row'));
if (diff_index==null){var diff_index = [];}
if (row_ind==null){var row_ind = 0;}
if (Roww==null){var Roww = Row;}
Logger.log(row_ind+'\n'+Roww);
for (var i=Roww;i<array.length;i++){
var currTime = (new Date()).getTime();
if(currTime - startTime >= MAX_RUNNING_TIME){
Logger.log((currTime - startTime)/(1000*60));
Logger.log(i+'::'+row_ind);
cache.putAll({'diff_index': JSON.stringify(diff_index),'row_ind': JSON.stringify(row_ind),'Row': JSON.stringify(i-1)},21600);
ScriptApp.newTrigger('getNewData').timeBased().after(2 * 60 * 1000).create();
return;
} else {
Logger.log(i);
var existe = ArrayLib.indexOf(old_array, Column_old, array[i][Column]);
if (existe==-1){
Logger.log(row_ind+'!!!');
diff_index[row_ind]=i;
row_ind++;
}
}
}
cache.putAll({'diff_index': JSON.stringify(diff_index),'Row': JSON.stringify(Row),'compared': JSON.stringify(1)},21600);
} else {
Logger.log('file not downloaded yet or already compared');
}
}
function getColumn(array,searchString){
for (var i = 0; i < array.length; i++) {
for (var j = 0; j < array[i].length; j++) {
if (array[i][j] == searchString) {
var Row = i+1;
var Column = j;
break;
}
}
if (Row != undefined){
break;
}
}
return {Row: Row, Column: Column};
}