My website becomes unresponsive when dealing 1000 rows excel file - json

I am uploading data from an excel file into my website using input html button.
and then convert the data into json and then I map it with local external metadata.
Finally view it using the id.
My website becomes unresponsive & sometimes takes a lot of time processing. Please help
function ExportToTable() {
var regex = /^([a-zA-Z0-9\s_\\.\-:()])+(.xlsx|.xls)$/;
/*Checks whether the file is a valid excel file*/
if (regex.test($("#excelfile").val().toLowerCase())) {
var xlsxflag = false; /*Flag for checking whether excel is .xls format or .xlsx format*/
if ($("#excelfile").val().toLowerCase().indexOf(".xlsx") > 0) {
xlsxflag = true;
}
/*Checks whether the browser supports HTML5*/
if (typeof (FileReader) != "undefined") {
var reader = new FileReader();
reader.onload = function (e) {
var data = e.target.result;
/*Converts the excel data in to object*/
if (xlsxflag) {
var workbook = XLSX.read(data, { type: 'binary' });
}
else {
var workbook = XLS.read(data, { type: 'binary' });
}
/*Gets all the sheetnames of excel in to a variable*/
var sheet_name_list = workbook.SheetNames;
console.log(sheet_name_list);
var cnt = 0; /*This is used for restricting the script to consider only first
sheet of excel*/
sheet_name_list.forEach(function (y) { /*Iterate through all sheets*/
/*Convert the cell value to Json*/
if (xlsxflag) {
var exceljson = XLSX.utils.sheet_to_json(workbook.Sheets[y]);
}
else {
var exceljson = XLS.utils.sheet_to_row_object_array(workbook.Sheets[y]);
}
//Download & View Subscriptions
if (exceljson.length > 0 && cnt == 1) {
metadata = [];
fetch("metadata.json")
.then(response => response.json())
.then(json => {
metadata = json;
console.log(metadata);
user_metadata1 = [], obj_m_processed = [];
for (var i in exceljson) {
var obj = { email: exceljson[i].email, name: exceljson[i].team_alias, id: exceljson[i].autodesk_id };
for (var j in metadata) {
if (exceljson[i].email == metadata[j].email) {
obj.GEO = metadata[j].GEO;
obj.COUNTRY = metadata[j].COUNTRY;
obj.CITY = metadata[j].CITY;
obj.PROJECT = metadata[j].PROJECT;
obj.DEPARTMENT = metadata[j].DEPARTMENT;
obj.CC=metadata[j].CC;
obj_m_processed[metadata[j].email] = true;
}
}
obj.GEO = obj.GEO || '-';
obj.COUNTRY = obj.COUNTRY || '-';
obj.CITY = obj.CITY || '-';
obj.PROJECT = obj.PROJECT || '-';
obj.DEPARTMENT = obj.DEPARTMENT || '-';
obj.CC = obj.CC || '-';
user_metadata1.push(obj);
}
for (var j in metadata) {
if (typeof obj_m_processed[metadata[j].email] == 'undefined') {
user_metadata1.push({ email: metadata[j].email, name: metadata[j].name, id: metadata[j].autodesk_id,
GEO: metadata[j].GEO,
COUNTRY : metadata[j].COUNTRY,
CITY : metadata[j].CITY,
PROJECT : metadata[j].PROJECT,
DEPARTMENT : metadata[j].DEPARTMENT,
CC:metadata[j].CC
});
}
}
document.getElementById("headings4").innerHTML = "MetaData Mapping";
BindTable(user_metadata1, '#user_metadata1
cnt++;
});
$('#exceltable').show();
}
if (xlsxflag) {/*If excel file is .xlsx extension than creates a Array Buffer from excel*/
reader.readAsArrayBuffer($("#excelfile")[0].files[0]);
}
else {
reader.readAsBinaryString($("#excelfile")[0].files[0]);
}
}
else {
alert("Sorry! Your browser does not support HTML5!");
}
}
else {
alert("Please upload a valid Excel file!");
}
}
Here is how the json is bind after mapping metadata
function BindTable(jsondata, tableid) {/*Function used to convert the JSON array to Html Table*/
var columns = BindTableHeader(jsondata, tableid); /*Gets all the column headings of Excel*/
for (var i = 0; i < jsondata.length; i++) {
var row$ = $('<tr/>');
for (var colIndex = 0; colIndex < columns.length; colIndex++) {
var cellValue = jsondata[i][columns[colIndex]];
if (cellValue == null)
cellValue = "";
row$.append($('<td/>').html(cellValue));
}
$(tableid).append(row$);
}
}
function BindTableHeader(jsondata, tableid) {/*Function used to get all column names from JSON and bind the html table header*/
var columnSet = [];
var headerTr$ = $('<tr/>');
for (var i = 0; i < jsondata.length; i++) {
var rowHash = jsondata[i];
for (var key in rowHash) {
if (rowHash.hasOwnProperty(key)) {
if ($.inArray(key, columnSet) == -1) {/*Adding each unique column names to a variable array*/
columnSet.push(key);
headerTr$.append($('<th/>').html(key));
}
}
}
}
$(tableid).append(headerTr$);
return columnSet;
}

Related

How do you overwrite data instead of appendRow?

Currently the code below appends new data to my spreadsheet every time it runs and I have to manually remove the data before running.
{
var ss = SpreadsheetApp.openById("blank");
var sheetName = ss.getSheetByName("Team Database");
var url = "blank";
var response = JSON.parse(UrlFetchApp.fetch(url));
var current = 1;
if(response.returnData && response.returnData.equipos.length > 0)
{
for(var team in response.returnData.equipos)
{
if(response.returnData.equipos[team].members.length > 0)
{
var i = 0;
while(i < response.returnData.equipos[team].members.length)
{
console.log(response.returnData.equipos[team].name);
console.log(response.returnData.equipos[team].members[i].userId);
sheetName.appendRow([response.returnData.equipos[team].name, response.returnData.equipos[team].members[i].userId]);
i++;
}
}
}
}
if(team.length > 0)
{
sheetName.getRange('D2').setValue('=NOW()');
sheetName.getRange('D1').setValue(sheetName.getRange('D2').getValue());
sheetName.getRange('D2').clear();
}
}
Is there a function that would overwrite it instead like: sheetName.overwriteRow([response.returnData.equipos[team].name, response.returnData.equipos[team].members[i].userId]);?
function myfunc() {
var ss = SpreadsheetApp.openById("blank");
var sh = ss.getSheetByName("Team Database");
var url = "blank";
var response = JSON.parse(UrlFetchApp.fetch(url));
var current = 1;
let oA = [];
if (response.returnData && response.returnData.equipos.length > 0) {
for (var team in response.returnData.equipos) {
if (response.returnData.equipos[team].members.length > 0) {
var i = 0;
while (i < response.returnData.equipos[team].members.length) {
console.log(response.returnData.equipos[team].name);
console.log(response.returnData.equipos[team].members[i].userId);
oA.push([response.returnData.equipos[team].name, response.returnData.equipos[team].members[i].userId]);
i++;
}
}
}
}
if (team.length > 0) {
sh.getRange('D2').setValue('=NOW()');
sh.getRange('D1').setValue(sh.getRange('D2').getValue());
sh.getRange('D2').clear();
sh.getRange(3,1,sh.getLastRow() - 2, sh.getLastColumn()).clearContent();
sh.getRange(3,1,oA.length, oA[0].length).setValues(oA);
}
}

Why google spreadsheets scripts can't sort?

function SUMMARIZE_TOGGL_ENTRIES(proj, desc, time, taskDates, uID) {
if (!desc.map || !time.map || !taskDates.map || !proj.map) {
return 'INPUT HAS TO BE MAP';
}
if (desc.length != time.length || time.length != taskDates.length || proj.length != taskDates.length) {
return 'WRONG INPUT ARRAYS LEN';
}
var resObj = {'NO_DESCRIPTION': '00:00:00'};
var keys = [];
var result = [];
for (var i = 1; i < desc.length; i++) {
var tmpKey = createKey(proj[i], desc[i], uID[i]);
console.log('tmpKey',tmpKey);
if (resObj[tmpKey]) {
resObj[tmpKey] = formatTime(timestrToSec(resObj[tmpKey]) + timestrToSec(time[i]));
} else {
resObj[tmpKey] = formatTime(timestrToSec(time[i]));
}
}
keys = Object.keys(resObj);
var b=0;
for (b; b < keys.length; b++) {
var tmp = [];
var key = keys[b].split('__')[1];
console.log('keys',keys[b].split('__'));
tmp.push(keys[b].split('__')[0]);
tmp.push(key);
tmp.push(resObj[keys[b]]);
tmp.push(taskLastDate(keys[b], desc, taskDates, proj, uID))
tmp.push(keys[b].split('__')[2]);
if(tmp[3] != '01-01-1991 01:01:01' && tmp[3] != ''){
result.push(tmp);
}
}
return result.sort(function (a,b){
var ad = new Date(a[3].split(' ')[0].replace(/-/g,'.'));
var bd = new Date(b[3].split(' ')[0].replace(/-/g,'.'));
if (ad > bd) {
return 1;
}
if (ad < bd) {
return -1;
}
return 0;
});
}
as you can see I'm trying to return a sorted array of arrays. But it don't returns the same array. If i lunch this func on my PC it works. so what's the problem? Does anyone have an idea?
result image
Seems like your sort function will always return 0. Change to
return result.sort(function (a,b){
var ad = new Date(a[3].split(' ')[0].replace(/-/g,'.'));
var bd = new Date(b[3].split(' ')[0].replace(/-/g,'.'));
if (ad > bd) {
return 1;
} else if (ad < bd) {
return -1;
} else {
return 0;
}
});
and see if that works?

While exporting as csv file, file is getting downloaded but with name download and with no extension

While exporting as csv file, file is getting downloaded but with name as download and with no extension,
I tried searching solution but didn't got any.
below is the code. If I rename the file and save as csv it shows the correct data
_addPrintButton: function() {
var me = this;
this.down('#print_button_box').add( {
xtype: 'rallybutton',
itemId: 'print_button',
text: 'Export to Excel',
disabled: false,
margin: '20 10 10 0',
region: "right",
handler: function() {
me._onClickExport();
}
});
},
_onClickExport: function(){
var grid = this.down('#grid_box');
var data = this._getCSV(grid.items.items[0]);
window.location = 'data:text/csv;charset=utf8,' + encodeURIComponent(data);
//Ext.getBody().unmask();
},
_getCSV: function (grid) {
var cols = grid.columns;
var store = grid.store;
var data = '';
var that = this;
_.each(cols, function(col, index) {
data += that._getFieldTextAndEscape(col.text) + ',';
});
data += "\r\n";
_.each(that.records, function(record) {
_.each(cols, function(col, index) {
var text = '';
var fieldName = col.dataIndex;
text = record[fieldName];
if (text || text == 0) {
//text = record[fieldName];
data += that._getFieldTextAndEscape(text) + ',';
}
/*else if (fieldName === "Project" ) {
text = record[fieldName];
}
else if (fieldName === "Case") {
var size = _.size(record[fieldName]);
for (var i = 0; i < size; i++){
text = record[fieldName][i]
}
}*/
});
data += "\r\n";
});
return data;
},
_getFieldTextAndEscape: function(fieldData) {
var string = this._getFieldText(fieldData);
return this._escapeForCSV(string);
},
_getFieldText: function(fieldData) {
var text;
if (fieldData === null || fieldData === undefined) {
text = '';
} else if (fieldData._refObjectName) {
text = fieldData._refObjectName;
}else {
text = fieldData;
}
return text.toString();
},
_escapeForCSV: function(string) {
if (string.match(/,/)) {
if (!string.match(/"/)) {
string = '"' + string + '"';
} else {
string = string.replace(/,/g, '');
}
}
return string;
},
You're searching at the wrong place.
The web server has to add HTTP header Content-Disposition to indicate the recommended file name, e.g.
Content-Disposition: attachment; filename="correct_data.csv"

Can I increase QUOTA_BYTES_PER_ITEM in Chrome?

Is there any way to increase the chrome.storage.sync.QUOTA_BYTES_PER_ITEM ?
For me, the default 4096 Bytes is a little bit short.
I tried to execute
chrome.storage.sync.QUOTA_BYTES_PER_ITEM = 8192;
However, it seems that the actual limit doesn't change.
How can I do this?
No, QUOTA_BYTES_PER_ITEM is there for reference only; it is not a settable value. You could use the value of QUOTA_BYTES_PER_ITEM to split an item up into multiple items, though:
function syncStore(key, objectToStore, callback) {
var jsonstr = JSON.stringify(objectToStore);
var i = 0;
var storageObj = {};
// split jsonstr into chunks and store them in an object indexed by `key_i`
while(jsonstr.length > 0) {
var index = key + "_" + i++;
// since the key uses up some per-item quota, see how much is left for the value
// also trim off 2 for quotes added by storage-time `stringify`
var valueLength = chrome.storage.sync.QUOTA_BYTES_PER_ITEM - index.length - 2;
// trim down segment so it will be small enough even when run through `JSON.stringify` again at storage time
var segment = jsonstr.substr(0, valueLength);
while(JSON.stringify(segment).length > valueLength)
segment = jsonstr.substr(0, --valueLength);
storageObj[index] = segment;
jsonstr = jsonstr.substr(valueLength);
}
// store all the chunks
chrome.storage.sync.set(storageObj, callback);
}
Then write an analogous fetch function that fetches by key and glues the object back together.
just modify answer of #apsilliers
function syncStore(key, objectToStore) {
var jsonstr = JSON.stringify(objectToStore);
var i = 0;
var storageObj = {};
// split jsonstr into chunks and store them in an object indexed by `key_i`
while(jsonstr.length > 0) {
var index = key + "_" + i++;
// since the key uses up some per-item quota, see how much is left for the value
// also trim off 2 for quotes added by storage-time `stringify`
const maxLength = chrome.storage.sync.QUOTA_BYTES_PER_ITEM - index.length - 2;
var valueLength = jsonstr.length;
if(valueLength > maxLength){
valueLength = maxLength;
}
// trim down segment so it will be small enough even when run through `JSON.stringify` again at storage time
//max try is QUOTA_BYTES_PER_ITEM to avoid infinite loop
var segment = jsonstr.substr(0, valueLength);
for(let i = 0; i < chrome.storage.sync.QUOTA_BYTES_PER_ITEM; i++){
const jsonLength = JSON.stringify(segment).length;
if(jsonLength > maxLength){
segment = jsonstr.substr(0, --valueLength);
}else {
break;
}
}
storageObj[index] = segment;
jsonstr = jsonstr.substr(valueLength);
}
also function to read each partition and merge again
function syncGet(key, callback) {
chrome.storage.sync.get(key, (data) => {
console.log(data[key]);
console.log(typeof data[key]);
if(data != undefined && data != "undefined" && data != {} && data[key] != undefined && data[key] != "undefined"){
const keyArr = new Array();
for(let i = 0; i <= data[key].count; i++) {
keyArr.push(`${data[key].prefix}${i}`)
}
chrome.storage.sync.get(keyArr, (items) => {
console.log(data)
const keys = Object.keys( items );
const length = keys.length;
let results = "";
if(length > 0){
const sepPos = keys[0].lastIndexOf("_");
const prefix = keys[0].substring(0, sepPos);
for(let x = 0; x < length; x ++){
results += items[`${prefix }_${x}`];
}
callback(JSON.parse(results));
return;
}
callback(undefined);
});
} else {
callback(undefined);
}
});
}
it tested and it works for my case
this is a better version of #uncle bob's functions, working with manifest v3 (you can use it just like how you can use the normal sync.set or sync.get function)
NOTE: it only works with JSONs (arrays and objects) since a string shouldn't be that long
let browserServices;
if (typeof browser === "undefined") {
browserServices = chrome;
} else {
browserServices = browser;
}
function syncSet(obj = {}) {
return new Promise((resolve, reject) => {
var storageObj = {};
for (let u = 0; u < Object.keys(obj).length; u++) {
const key = Object.keys(obj)[u];
const objectToStore = obj[key]
var jsonstr = JSON.stringify(objectToStore);
var i = 0;
// split jsonstr into chunks and store them in an object indexed by `key_i`
while (jsonstr.length > 0) {
var index = key + "USEDTOSEPERATE" + i++;
// since the key uses up some per-item quota, see how much is left for the value
// also trim off 2 for quotes added by storage-time `stringify`
const maxLength = browserServices.storage.sync.QUOTA_BYTES_PER_ITEM - index.length - 2;
var valueLength = jsonstr.length;
if (valueLength > maxLength) {
valueLength = maxLength;
}
// trim down segment so it will be small enough even when run through `JSON.stringify` again at storage time
//max try is QUOTA_BYTES_PER_ITEM to avoid infinite loop
var segment = jsonstr.substring(0, valueLength);
var jsonLength = JSON.stringify(segment).length;
segment = jsonstr.substring(0, valueLength = (valueLength - (jsonLength - maxLength) - 1));
for (let i = 0; i < browserServices.storage.sync.QUOTA_BYTES_PER_ITEM; i++) {
jsonLength = JSON.stringify(segment).length;
if (jsonLength > maxLength) {
segment = jsonstr.substring(0, --valueLength);
} else {
break;
}
}
storageObj[index] = segment;
jsonstr = jsonstr.substring(valueLength, Infinity);
}
}
chrome.storage.sync.set(storageObj).then(() => {
resolve()
})
})
}
function syncGet(uniqueKeys = []) {
return new Promise((resolve, reject) => {
browserServices.storage.sync.get(null).then((data) => {
const keyArr = Object.keys(data).filter(e => uniqueKeys.filter(j => e.indexOf(j) == 0).length > 0)
browserServices.storage.sync.get(keyArr).then((items) => {
var results = {};
for (let i = 0; i < uniqueKeys.length; i++) {
const uniqueKey = uniqueKeys[i];
const keysFiltered = keyArr.filter(e => e.split("USEDTOSEPERATE")[0] == uniqueKey)
if (keysFiltered.length > 0) {
results[uniqueKey] = ""
for (let x = 0; x < keysFiltered.length; x++) {
results[uniqueKey] += items[`${keysFiltered[x]}`];
}
results[uniqueKey] = JSON.parse(results[uniqueKey])
}
}
resolve(results)
});
});
})
}
example of usage:
syncSet({
"keyTest": ["a lot of text"],
"keyTest1": ["a lot of text"]
}
)
syncGet(["keyTest","keyTest1"]).then(results=>console.log(results))
// {keyTest:["a lot of text"],keyTest1:["a lot of text"]}

json stringify in IE 8 gives run time error Object property or method not supported

/* Problem description- I am using json stringify method to convert an javascript array to string in json notation.However I get an error message that 'Object property or method not supported' at line
hidden.value = JSON.stringify(jsonObj);
This should work as stringify is supported in IE8.Please advise
Full code below */
function getgridvalue() {
var exportLicenseId;
var bolGrossQuantity;
var bolNetQuantity;
var totalBolGrossQty =0 ;
var totalBolNetQty =0;
var jsonObj = []; //declare array
var netQtyTextBoxValue = Number(document.getElementById("<%= txtNetQty.ClientID %>").value);
var atLeastOneChecked = false;
var gridview = document.getElementById("<%= ExporterGrid.ClientID %>"); //Grab a reference to the Grid
for (i = 1; i < gridview.rows.length; i++) //Iterate through the rows
{
if (gridview.rows[i].cells[0].getElementsByTagName("input")[0] != null && gridview.rows[i].cells[0].getElementsByTagName("input")[0].type == "checkbox")
{
if (gridview.rows[i].cells[0].getElementsByTagName("input")[0].checked)
{
atLeastOneChecked = true;
exportLicenseId = gridview.rows[i].cells[8].getElementsByTagName("input")[0].value;
bolNetQuantity = gridview.rows[i].cells[5].getElementsByTagName("input")[0].value;
if (bolNetQuantity == "") {
alert('<%= NetQuantityMandatory %>');
return false;
}
if (!isNumber(bolNetQuantity)) {
alert('<%= NetQuantityNumber %>');
return false;
}
totalBolNetQty += Number(bolNetQuantity);
jsonObj.push({ ExportLicenseId: Number(exportLicenseId), BolNetQuantity: Number(bolNetQuantity) });
}
}
}
if (gridview.rows.length > 2 && !atLeastOneChecked)
{
alert('<%= SelectMsg %>');
return false;
}
if (totalBolNetQty != 0 && netQtyTextBoxValue != totalBolNetQty)
{
alert('<%= NetQuantitySum %>');
return false;
}
var hidden = document.getElementById('HTMLHiddenField');
// if (!this.JSON) {
// this.JSON = {};
// }
var JSON = JSON || {};
if (hidden != null) {
hidden.value = JSON.stringify(jsonObj);
}
}
Use the F12 Developer Tools to check the browser mode. The JSON object exists, but has no methods in IE7 mode. Use the json2 library as a fallback.
json2.js