I have noticed that the script is significantly faster when I comment out the line that calls setValues(). Not that it's a super big issue, but I am interested in the different ways (if any) that I could speed the process up. If anyone has some insight, I would greatly appreciate it! It would also help me see where I could make similar optimizations in other parts of my script.
Here is the code in question:
// import new value(s)
if ((numRaw - numDk) > 0) {
var iDate;
var lastRow = getLastDataRow(earSheet, columnToLetter(earSheet.getRange("EAR_DATES_RNG").getColumn()));
var distName = impSheet.getRange("A1").getValue().toString();
var defaultDept = SpreadsheetApp.getActiveSpreadsheet().getSheetByName("All Lists").getRange("A2").getValue().toString();
impData.forEach(function (entry) {
iDate = new Date(entry[0]);
var newEar = earSheet.getRange("A" + lastRow + ":G" + lastRow).getValues(); // init copy
if (currData.get(iDate.getTime()) == null) {
newEar[0][0] = entry[0];
newEar[0][1] = distName;
newEar[0][6] = entry[1];
newEar[0][3] = "Royalties";
newEar[0][4] = defaultDept;
newEar[0][5] = "NOT SPECIFIED";
earSheet.getRange("A" + lastRow + ":G" + lastRow).setValues(newEar);
Logger.log("ADDED: " + entry[1] + " to row: " + lastRow);
addedNewData++;
lastRow++;
}
else {
Logger.log("EXISTING DATA FOUND: " + currData.get(iDate.getTime()));
}
});
}
Using Matt and Cooper's suggestions, I came up with this revised code which is significantly faster:
if ((numRaw - numDk) > 0) {
var iDate;
var prevLastRow = getLastDataRow(earSheet, columnToLetter(earSheet.getRange("EAR_DATES_RNG").getColumn()));
var currLastRow = getLastDataRow(earSheet, columnToLetter(earSheet.getRange("EAR_DATES_RNG").getColumn()));
var rangeSize = numRaw;
var distName = impSheet.getRange("A1").getValue().toString();
var defaultDept = SpreadsheetApp.getActiveSpreadsheet().getSheetByName("All Lists").getRange("A2").getValue().toString();
var newEar = earSheet.getRange("A" + prevLastRow + ":G" + (prevLastRow + rangeSize - 1)).getValues(); // init copy
impData.forEach(function (entry) {
iDate = new Date(entry[0]);
if (currData.get(iDate.getTime()) == null) {
newEar[currLastRow-2][0] = entry[0];
newEar[currLastRow-2][1] = distName;
newEar[currLastRow-2][6] = entry[1];
newEar[currLastRow-2][3] = "Royalties";
newEar[currLastRow-2][4] = defaultDept;
newEar[currLastRow-2][5] = "NOT SPECIFIED";
Logger.log("ADDED: " + entry[1] + " to row: " + currLastRow);
addedNewData++;
currLastRow++;
}
else {
Logger.log("EXISTING DATA FOUND: " + currData.get(iDate.getTime()));
}
});
earSheet.getRange("A" + prevLastRow + ":G" + (prevLastRow + rangeSize - 1)).setValues(newEar);
}```
Related
The script doesn't throw any errors, but rarely completely works - i.e. complete successfully with all of the expected data in the destination tab. The results breakdown is generally:
no results in the destination sheet - this happens ~50-75% of the time
all of the results in the destination sheet, except in cell A1 - ~25% of the time
100% completely works - ~15-25% of the time
code snippet of the batchupdate() call
var data = [
{
range: (ss.getSheetName() + "!A1:AQ" + valueArray.length)
,values: valueArray
}
];
const resource = {
valueInputOption: "RAW"
,data: data
};
Logger.log("request = " + JSON.stringify(resource)
+ "\n" + "valueArray = " + valueArray.length
);
Logger.log(" Sheets.Spreadsheets.Values.batchUpdate(params, batchUpdateValuesRequestBody) ");
var response = Sheets.Spreadsheets.Values.batchUpdate(resource, spreadsheetId);
Logger.log("response = " + response.toString());
and the response
response = {
"totalUpdatedRows": 37776,
"responses": [{
"updatedCells": 1482389,
"updatedRange": "BatchUpdateDestination!A1:AP37776",
"updatedColumns": 42,
"spreadsheetId": "adahsdassadasdsadaasdasdasdasdasdasdasdasdas",
"updatedRows": 37776
}
],
"spreadsheetId": "adahsdassadasdsadaasdasdasdasdasdasdasdasdas",
"totalUpdatedCells": 1482389,
"totalUpdatedSheets": 1,
"totalUpdatedColumns": 42
}
Its obviously a very large dataset, but I've pruned the destination spreadsheet to ensure there is ample room for the data, and from earlier testing, I believe that a specific size error would be returned if that was the blocker.
How can I troubleshoot, or better yet, prevent these incomplete executions? is there any way to inspect the batch jobs that these requests initiate?
Answering my own question...
After toiling with this a little more, I couldn't figure out any way to troublshooting or inspect the odd, seemingly successfully batchUpdate() jobs. Thus, I resorted to batching the batchUpdate() calls into batches of 15000. This seems to work consistently, though maybe a bit slower:
// This is the very large 2D array that is populated elsewhere
var valueArray = [];
var maxRows = valueArray.length;
var maxCols = valueArray[0].length;
var batchSize = 15000;
var lastBatchSize = 1;
for (var currentRowCount = 1; currentRowCount <= maxRows; ++currentRowCount) {
if( currentRowCount % batchSize == 0
|| currentRowCount == maxRows
)
{
Logger.log("get new valuesToSet");
valuesToSet = valueArray.slice(lastBatchSize - 1, currentRowCount -1);
var data = [
{
range: (ss.getSheetName() + "!A" + lastBatchSize + ":AQ" + (lastBatchSize + valuesToSet.length))
,values: valuesToSet
}
];
const resource = {
valueInputOption: "RAW"
,data: data
};
Logger.log("request = " + JSON.stringify(resource).slice(1, 100)
+ "\n" + "valuesToSet.length = " + valuesToSet.length
);
try {
var checkValues = null;
var continueToNextBatch = false;
for (var i = 1; i <= 3; ++i) {
Logger.log("try # = " + i
+ "\n" + " continueToNextBatch = " + continueToNextBatch
+ "\n" + " make the batchUpdate() request, then sleep for 5 seconds, then check if there are values in the target range."
+ "\n" + " if no values, then wait 5 seconds, check again."
+ "\n" + " if still not values after 3 tries, then resubmit the batchUpdate() requestion and recheck values"
+ "\n" + "range to check = " + "A" + lastBatchSize + ":AQ" + lastBatchSize
);
Logger.log(" Sheets.Spreadsheets.Values.batchUpdate(params, batchUpdateValuesRequestBody) ");
var response = Sheets.Spreadsheets.Values.batchUpdate(resource, spreadsheetId);
Logger.log("response = " + response.toString());
/// loop and check for data in newly written range
for (var checks = 1; checks <= 3; ++checks) {
Utilities.sleep(5000);
var checkValues = ss.getRange(("A" + lastBatchSize + ":AQ" + lastBatchSize)).getValues();
Logger.log("new cell populated - checks # = " + checks
+ "\n" + "range to check = " + "A" + lastBatchSize + ":AQ" + lastBatchSize
+ "\n" + "checkValues.length = " + checkValues.length
+ "\n" + "checkValues = " + checkValues
);
if(checkValues.length > 1)
{
Logger.log("checkValues.length > 1, so continue to next batch"
+ "\n" + "range to check = " + "A" + lastBatchSize + ":AQ" + lastBatchSize
+ "\n" + "checkValues.length = " + checkValues.length
+ "\n" + "checkValues = " + checkValues
);
continueToNextBatch = true;
continue;
}
else
{
Logger.log("checkValues.length is still not > 1, so try the request again"
+ "\n" + "range to check = " + "A" + lastBatchSize + ":AQ" + lastBatchSize
);
}
}
if(continueToNextBatch)
{
continue;
}
}
}
catch (e) {
console.error("range.setValues(valuesToSet) - yielded an error: " + e
+ "\n" + "valuesToSet = " + valuesToSet.length
+ "\n" + "maxRows = " + maxRows
+ "\n" + "maxCols = " + maxCols
+ "\n" + "currentRowCount = " + currentRowCount
+ "\n" + "current range row start (lastBatchSize) = " + lastBatchSize
+ "\n" + "current range row end (j - lastBatchSize) = " + (currentRowCount - lastBatchSize)
);
}
lastBatchSize = currentRowCount;
}
}
I am using node.js with my WebStorm IDE to parse a large JSON file (~500 megabytes). Here is my code:
fs = require("fs");
fs.readFile('C:/Users/.../Documents/AAPL.json', 'utf8', function (err,data) {
for (i = 0; i < 1000; i++) {
var hex = JSON.parse(data)[i]._source.layers.data["data.data"];
var askPrice = parseInt(hex.substring(215, 239).split(":").reverse().join(""),16);
var bidPrice = parseInt(hex.substring(192, 215).split(":").reverse().join(""),16);
var symbol = hex.substring(156, 179);
var timestamp = hex.substring(132, 155);
var askSize = hex.substring(240, 251);
var bidSize = hex.substring(180, 191);
var price = String((+bidPrice+askPrice)/2);
var realprice = price.slice(0, price.length - 4) + "." + price.slice(price.length - 4);
function hex2a(hexx) {
var hex = hexx.toString();
var str = '';
for (var i = 0; i < hex.length; i += 2)
str += String.fromCharCode(parseInt(hex.substr(i, 2), 16));
return str;
}
if(JSON.parse(data)[i]._source.layers.data["data.len"] == 84 && realprice.length == 8 && +realprice <154 && +realprice >145) {
console.log(i + " " + hex2a(symbol.replace(/:/g, "")) + " sold for " + realprice + " at " + parseInt(timestamp.split(":").reverse().join(""), 16));
}
}
});
The problem I am running into however is that my IDE is parsing this file at an extremely slow speed, roughly 1 iteration a second. I do not think this is because I have a slow computer, for I have a high end rig with a core i7 7700k and a gtx 1070. I tried executing the code in the console with the same result. I tried trimming down the code and again I achieved the same speed:
fs = require("fs");
fs.readFile('C:/Users/Brandt Winkler Prins/Documents/AAPL.json', 'utf8', function (err,data) {
for (i = 0; i < 12000; i++) {
var hex = JSON.parse(data)[i]._source.layers.data["data.data"];
var askPrice = parseInt(hex.substring(215, 239).split(":").reverse().join(""),16);
var bidPrice = parseInt(hex.substring(192, 215).split(":").reverse().join(""),16);
var price = String((+bidPrice+askPrice)/2);
var realprice = price.slice(0, price.length - 4) + "." + price.slice(price.length - 4);
if(+realprice <154 && +realprice >145) {
console.log(realprice);
}
}
});
How should I execute my code to get my data as fast as possible?
You're running JSON.parse(data) every iteration, that might take quite some time for a 500MB json file.
The solution would be to move it out of the loop and reuse the parsed object:
var obj = JSON.parse(data);
for (...
I have an input textfield on the screen and I have it set to the text that a user enters is saved as a variable and is later called upon o be displayed in a dynamic text box. It's kinda like a high score kind of system, but with multiple variables.
Here is the frame actions where the variables are being set (at least I think they are)
button.addEventListener(MouseEvent.CLICK, fl_MouseClickHandler);
function fl_MouseClickHandler(event:MouseEvent):void
{
var data:String = username.text + " " + date.text + " " + company.text;
var file:FileReference = new FileReference();
file.save(data, username.text + " " + date.text + " " + company.text + ".txt");
}
button.addEventListener(MouseEvent.CLICK, fl_ClickToGoToNextFrame_8);
function fl_ClickToGoToNextFrame_8(event:MouseEvent):void
{
nextFrame();
}
var nameperson = username.text;
var dateperson = date.text;
var companyperson = company.text;
And are the actions where I'm trying to display the variables in another frame:
var nScore:Number = 0;
for(var i:Number = 0; i < aQuestions.length; i++)
{
if(aUserAnswers[i].toUpperCase() == aCorrectAnswers[i].toUpperCase())
{
nScore++;
}
if(i == aQuestions.length - 1)
{
score_txt.text = nScore.toString();
}
}
endresult_name.text = nameperson;
endresult_date.text = dateperson;
I believe the problem is that you set these variables straight when you get to your 1st frame (the text inputs are empty at that point). You need to set them after the user have filled them out and clicked on the button:
var nameperson:String;
var dateperson:String;
var companyperson:String;
function fl_MouseClickHandler(event:MouseEvent):void
{
var data:String = username.text + " " + date.text + " " + company.text;
var file:FileReference = new FileReference();
file.save(data, username.text + " " + date.text + " " + company.text + ".txt");
nameperson = username.text;
dateperson = date.text;
companyperson = company.text;
}
I have some code not written by me that I'm trying to compile.
public static function getUserInfoObject(info:Array) : Object {
var lastBattleTime:Number = info[7];
var listLength:Number = info[8];
var list:Array = info.slice(9,9 + listLength);
var achievesLength:Number = info[9 + listLength];
var achievements:Array = info.slice(10 + listLength,10 + listLength + achievesLength);
var statsLength:Number = info[10 + listLength + achievesLength];
var stats:Array = info.slice(11 + listLength + achievesLength,11 + listLength + achievesLength + statsLength);
var commonInfo:Array = info.slice(11 + listLength + achievesLength + statsLength,11 + listLength + achievesLength + statsLength + 8);
return
{
"uid":info[0],
"name":info[1],
"chatRoster":info[2],
"status":info[3],
"displayName":info[5],
"list":list,
"achievements":achievements,
"stats":stats,
"commonInfo":commonInfo,
"creationTime":App.utils.locale.longDate(info[6]),
"lastBattleTime":(lastBattleTime == 0?"":App.utils.locale.longDate(lastBattleTime) + " " + App.utils.locale.longTime(lastBattleTime))
};
}
It gives me this error: 1078: Label must be a simple identifier. in every line in return.
Am I blind or dumb or this code is bad?
You should start your return statement with the curly brace, not with new line:
public static function getUserInfoObject(info:Array) : Object {
return { // <-Here
};
}
I am stuck.. Error is thrown # Line 63 and sometimes at line 50. Both using the appendTableRow() method. I can't find anything wrong.
row3.appendTableCell(entryDesc)
Link to generated file: Link
Execution Transcript: Link
I am new so if you notice any "bad practices" feel free to aim a finger.
// Import data from the Calendar to the timesheet document
function importDataToTS (dateStart,dateFinish,doc) {
if (!dateStart) {
var dateStart = new Date('January 1, 2014');
}
var cal = CalendarApp.getCalendarById('0k2s9lfibn50scj41gcuurovck#group.calendar.google.com')
var events = cal.getEvents(dateStart, dateFinish);
var oldDate = new Date(dateStart.getFullYear(), dateStart.getMonth(), dateStart.getDate() - 1);
var paragraph = "";
var totalHoursWorked = 0
// START --- Text element styles
// Date
var entryDateStyle = {};
entryDateStyle[DocumentApp.Attribute.BOLD] = true;
entryDateStyle[DocumentApp.Attribute.FONT_SIZE] = 18;
// Title
var entryTitleStyle = {};
entryTitleStyle[DocumentApp.Attribute.FONT_SIZE] = 14;
// entryTimes
var entryTimesStyle = {};
entryTimesStyle[DocumentApp.Attribute.BOLD] = true;
entryTimesStyle[DocumentApp.Attribute.FONT_SIZE] = 12;
// entryDescription
var entryDescriptionStyle = {};
entryDescriptionStyle[DocumentApp.Attribute.ITALIC] = true;
entryDescriptionStyle[DocumentApp.Attribute.FONT_SIZE] = 10;
// END --- Text element styles
var entriesTable = doc.appendTable();
Logger.log(entriesTable.getType());
for (var i = 0; i in events; i++) {
var entryDate = events[i].getStartTime();
if (i > 0) {
oldDate = (events[i-1].getStartTime());
}
// If it's a new day add a full width cell
if (entryDate.getDate() > oldDate.getDate()) {
var row1 = entriesTable.appendTableRow();
row1.appendTableCell(shortDate(entryDate,4));
//.setAttributes(entryDateStyle);
}
// Add title, start/end times & hours worked
// Add title, start/end times & hours worked
var entryTitle = events[i].getTitle();
Logger.log(i + ": " + entryTitle);
var entryTimes = shortTime(events[i].getStartTime(),2) + " - " + shortTime(events[i].getEndTime(),2);
Logger.log(i + ": " + entryTimes);
var entryHoursWorked = ((events[i].getEndTime() - events[i].getStartTime())/(1000*60*60)%24) + "hr(s)";
Logger.log(i + ": " + entryHoursWorked);
var row2 = entriesTable.appendTableRow();
row2.appendTableCell(entryTitle);
//.setAttributes(entryTitleStyle);
row2.appendTableCell(entryTimes + "\t\t" + entryHoursWorked);
//.setAttributes(entryTimesStyle);
// Add entry description
var entryDesc = (events[i].getDescription().length > 1) ? events[i].getDescription().toString() : "";
if (entryDesc.length > 1) {
var row3 = entriesTable.appendTableRow();
row3.appendTableCell();
row3.appendTableCell(entryDesc);
//.setAttributes(entryDescriptionStyle);
}
totalHoursWorked += entryHoursWorked;
if (i === (events.length - 1)) {
var lastRow = entriesTable.appendTableRow();
lastRow.appendTableCell("Total Hours: " + totalHoursWorked);
}
}
for (var i = 0; i in entriesTable; i++) {
for (var j = 0; j in entriesTable[i]; j++) {
Logger.log(i + ":" + j + " " + entriesTable[i][j].toString());
}
}
doc.appendTable(entriesTable);
}
shortDate() && shorttTime()
function shortDate(date,n) {
// Returns a date object as "MMMDD";
if (date) {
switch (n) {
case 1:
//Jul6
return Utilities.formatDate(date, "EST", "MMMdd")
break;
case 2:
//Jul 6
return Utilities.formatDate(date, "EST", "MMM dd")
break;
case 3:
//July 6
return Utilities.formatDate(date, "EST", "MMMM dd")
break;
case 4:
return Utilities.formatDate(date, "EST", "EEE, MMM dd")
break;
default:
//Full Date unchanged
return date;
}
}
}
function shortTime(date) {
//Returns time string formatted as "hh:mm"am/pm
//Still needs to be updated with Utilities.formatDate
if (date) {
var hours = (date.getHours() > 12) ? (date.getHours() - 12) : date.getHours();
var ampm = (date.getHours() > 12) ? "pm" : "am";
var minutes = (date.getMinutes() == 0) ? "00" : date.getMinutes();
var time = hours + ":" + minutes + ampm
return time.toString();
}
}
I found a solution, it appears that: body.appendTableCell(); doesn't handle line breaks "\n". When the script was importing a multi-line event description from the calendar I would get a "server error" message. Adding split('\n') to the description row solved the problem. This worked: body.appendTableCell(data).split("\n");
Finished code:
var entryDesc = (events[i].getDescription().length > 1) ? events[i].getDescription() : "";
if (entryDesc) {
var row3 = entriesTable.appendTableRow();
row3.appendTableCell("");
row3.appendTableCell(entryDesc.split("\n"))
.setAttributes(entryDescriptionStyle);
}
I don't have a complete answer but I thought it might be interesting in the mean time to show a version that works without the event description.
I changed the calculation of total time that didn't work either.
Code can be tested on any default calendar using test function.
function test(){
//dateStart,dateFinish,doc
var doc = DocumentApp.getActiveDocument();
var dateStart = new Date('January 1, 2014');
var dateFinish = new Date('April 1, 2014')
importDataToTS (dateStart,dateFinish,doc);
}
function importDataToTS (dateStart,dateFinish,doc) {
if (!dateStart) {
var dateStart = new Date('January 1, 2014');
}
var cal = CalendarApp.getDefaultCalendar();
var events = cal.getEvents(dateStart, dateFinish);
var oldDate = new Date(dateStart.getFullYear(), dateStart.getMonth(), dateStart.getDate() - 1);
var paragraph = "";
var totalHoursWorked = 0
// START --- Text element styles
// Date
var entryDateStyle = {};
entryDateStyle[DocumentApp.Attribute.BOLD] = true;
entryDateStyle[DocumentApp.Attribute.FONT_SIZE] = 18;
// Title
var entryTitleStyle = {};
entryTitleStyle[DocumentApp.Attribute.FONT_SIZE] = 14;
// entryTimes
var entryTimesStyle = {};
entryTimesStyle[DocumentApp.Attribute.BOLD] = true;
entryTimesStyle[DocumentApp.Attribute.FONT_SIZE] = 12;
// entryDescription
var entryDescriptionStyle = {};
entryDescriptionStyle[DocumentApp.Attribute.ITALIC] = true;
entryDescriptionStyle[DocumentApp.Attribute.FONT_SIZE] = 10;
// END --- Text element styles
var entriesTable = doc.appendTable();
Logger.log('events.length = '+events.length);
for (var i = 0; i <events.length; i++) {
var entryDate = events[i].getStartTime();
if (i > 0) {
oldDate = (events[i-1].getStartTime());
}
Logger.log('i = '+i);
// If it's a new day add a full width cell
if (entryDate.getDate() > oldDate.getDate()) {
var row1 = entriesTable.appendTableRow();
row1.appendTableCell(shortDate(entryDate,4))
.setAttributes(entryDateStyle);
}
// Add title, start/end times & hours worked
var entryTitle = events[i].getTitle();
var entryTimes = shortTime(events[i].getStartTime(),2) + " - " + shortTime(events[i].getEndTime(),2);
var entryHoursWorked = (events[i].getEndTime().getTime() - events[i].getStartTime().getTime())/(1000*60*60) + "hr(s)";
var row2 = entriesTable.appendTableRow();
row2.appendTableCell(entryTitle)
.setAttributes(entryTitleStyle);
row2.appendTableCell(entryTimes + "\t\t" + entryHoursWorked)
.setAttributes(entryTimesStyle);
// Add entry description
var entryDesc = (events[i].getDescription().length > 2) ? events[i].getDescription() : "";
totalHoursWorked += Number(entryHoursWorked.replace(/\D/g,''));
if (i === (events.length - 1)) {
var lastRow = entriesTable.appendTableRow();
lastRow.appendTableCell("Total Hours: " + totalHoursWorked+' Hours');
}
}
for (var i = 0; i in entriesTable; i++) {
for (var j = 0; j in entriesTable[i]; j++) {
Logger.log(i + ":" + j + " " + entriesTable[i][j].toString());
}
}
doc.saveAndClose();
}