Concatenate large dataset from multiple worksheets of a Google Sheet - google-apps-script

Question

I'm using the code below to concatenate certain sheets in my file into a sheet named "Master".
Currently, my script duplicates the header row, but I would like to prevent this. The data in all the sheets is in exactly the same format with the same header.
Each sheet has 58 columns and one of the sheets has 5000 rows.
Ideally I'd like the script to overwrite the "Master" sheet each time the script is run.
How can I modify my script to accomplish these goals?
function concatAllSheets()
{
var includedSheet = ['Virtue data - Norway - NOK', 'Virtue data - Sweden - SKK', 'Virtue data - Denmark - DKK', 'Virtue Data - GBP', 'Virtue data - EUR markets', 'Virtue data - Arabia - USD'];
var ss = SpreadsheetApp.getActive();
var allSheets = ss.getSheets();
var sheetName = 'Master'
var mother = ss.insertSheet(sheetName);
for(var i = 0; i < allSheets.length; i++)
{
var sht = allSheets[i];
if(includedSheet.indexOf(sht.getName()) > -1)
{
var rng = sht.getDataRange();
var rngA = rng.getValues();
for(var j = 0; j < rngA.length; j++)
{
var row = rngA[j];
mother.appendRow(row);
}
}
}
}

Answer 1

Rather than calling appendRow() for each row of data, which will really slow your script down, use a batch operation like setValues().
To do so, create an allData array to hold what will be the content of your "Master" sheet. As you iterate through the sheets, append their data into allData, and finally print it to the sheet.
As you're iterating, you can do a simple check of allData's length, to see if the header row is already there. If allData is empty, then there is obviously no header row.
function concatAllSheets()
{
var includedSheet=['Virtue data - Norway - NOK','Virtue data - Sweden - SKK','Virtue data - Denmark - DKK','Virtue Data - GBP','Virtue data - EUR markets','Virtue data - Arabia - USD'];
var ss=SpreadsheetApp.getActive();
var allSheets=ss.getSheets();
var sheetName='Master'
var mother=ss.insertSheet(sheetName);
var allData = [];
for(var i=0;i<allSheets.length;i++)
{
var sht=allSheets[i];
if(includedSheet.indexOf(sht.getName())>-1)
{
var rng=sht.getDataRange();
var rngA=rng.getValues();
if (allData.length == 0) // This will only ever be true on the first sheet copied
{
allData = rngA;
} else {
rngA.shift(); // Remove the first row
allData = allData.concat(rngA);
}
}
}
mother.getRange(1, 1, allData.length, allData[0].length).setValues(allData); // Use a batch operation to insert the data
}

Answer 2

Given the size of the dataset, you need to minimize reads and writes by using the batch methods Range#getValues and Range#setValues. You can also avoid iterating over the unnecessary sheets by only binding the ones in your array of names, using Array#forEach. This pattern also ensures that you collect all the data you intend - if there is a typo in your names array, or the sheet's name is changed unintentionally, this will throw an exception rather than silently not including the data.
function concatenateSheets() {
var sheetNames = ["name1", "name2", ...];
var ss = SpreadsheetApp.getActive();
var dest = ss.getSheetByName("someName");
var output = [], header = [];
// Assemble a single paste output from all the sheets.
sheetNames.forEach(function (name) {
var sheet = ss.getSheetByName(name);
if(!sheet)
throw new Error("Incorrect sheet name '" + name + "'");
var vals = sheet.getDataRange().getValues();
// Remove the header row.
header = vals.splice(0, 1);
// Append to existing output array.
output = [].concat(output, vals);
});
// Serialize output data array.
if(dest && output.length && output[0].length) {
// Remove all existing data values on the destination sheet.
// (Only necessary if the number of rows or columns can decrease.)
dest.getDataRange().clearContent();
// Prepend the header on the output data array.
output.unshift(header);
dest.getRange(1, 1, output.length, output[0].length).setValues(output);
}
}