Upload Excel using nodejs - mysql

I am uploading an Excel file and inserting into a mysql table. My code is working with Linux. But I want it should run with other also so I am predicting that Xls is not working.
How to change into CSV or any other way so we can upload the Excel or csv which support any version?
function getfileDetails(req, res) {
var sampleFile, fileInfo = {};
var date1=new Date();
var currdatetime = date1.getFullYear()+"-"+(date1.getMonth()+1)+"-"+date1.getDate()+" "+date1.getHours()+":"+date1.getMinutes()+":"+date1.getSeconds();
console.log(currdatetime);
var MyData = [];
if (!req.files) {
res.send('No files were uploaded.');
return;
}
sampleFile = req.files.fileInputXLSX1;
var datetimestamp = Date.now();
console.log("Uploaded -- ",sampleFile);
var fileExtn = sampleFile.name.split(".").pop();
var extractedFilename = sampleFile.name.slice(0, sampleFile.name.lastIndexOf('.'));
var userid=req.headers['userid'];
var uploadFileName = extractedFilename+'_'+userid+'_'+datetimestamp+'.'+fileExtn;
console.log("uploadFileName -- ",uploadFileName);
fileInfo = {
"name": uploadFileName,
"mimetype": sampleFile.mimetype
}
// Use the mv() method to place the file somewhere on your server
sampleFile.mv(__dirname+'/Details/'+uploadFileName, function(err) {
if (err) {
res.status(500).send(err);
}
var parseXlsx = require('excel');
parseXlsx(__dirname+'/Details/'+uploadFileName, function(err, data) {
if(err) throw err;
// data is an array of arrays
else{
if(data!=null)
{
var queryString= "Truncate table `details`;"
connection.query(queryString, function(err,result){
if(err) {
res.write(JSON.stringify(err));
res.end();
} else {
res.send('File uploaded!');
}
});
}
for (var index = 1; index < data.length; index++)
{
MyData.push(data[index][0],data[index][1],data[index][2],data[index][3], data[index][4], data[index][5],data[index][6],data[index][7],data[index][8],data[index][9]);
var queryString="INSERT INTO `details`(name,tname,fname,timestamp) VALUES ('"+data[index][0]+"','"+data[index][1]+"','"+data[index][2]+"','"+currdatetime+"')";
connection.query(queryString, function(err,result){
if(err) {
res.write(JSON.stringify(err));
res.end();
} else {
res.send('File uploaded!');
}
});
}
}
});
if(err) {
res.status(500);
res.send(err);
}
});
}
Please help ,I am New in Node.

hey can you try json2csv npm module. this will solve you problem #TB.M

This code is working fine, try this..
var xlsx = require('node-xlsx');
var fs = require('fs');
var obj = xlsx.parse(__dirname + '/new.xls'); // parses a file
var rows = [];
var writeStr = "";
//looping through all datarow
for(var i = 0; i < obj.length; i++)
{
var dataRow = obj[i];
//loop through all rows in the sheet
for(var j = 0; j < dataRow['data'].length; j++)
{
//add the row to the rows array
rows.push(dataRow['data'][j]);
}
}
//creates the csv string to write it to a file
for(var i = 0; i < rows.length; i++)
{
writeStr = writeStr + rows[i].join(",") + "\n";
}
//writes to a file, but you will presumably send the csv as a
//response instead
fs.writeFile(__dirname + "/data.csv", writeStr, function(err) {
if(err) {
return console.log(err);
}
console.log("data.csv was saved in the current directory!");
});

Related

My website becomes unresponsive when dealing 1000 rows excel file

I am uploading data from an excel file into my website using input html button.
and then convert the data into json and then I map it with local external metadata.
Finally view it using the id.
My website becomes unresponsive & sometimes takes a lot of time processing. Please help
function ExportToTable() {
var regex = /^([a-zA-Z0-9\s_\\.\-:()])+(.xlsx|.xls)$/;
/*Checks whether the file is a valid excel file*/
if (regex.test($("#excelfile").val().toLowerCase())) {
var xlsxflag = false; /*Flag for checking whether excel is .xls format or .xlsx format*/
if ($("#excelfile").val().toLowerCase().indexOf(".xlsx") > 0) {
xlsxflag = true;
}
/*Checks whether the browser supports HTML5*/
if (typeof (FileReader) != "undefined") {
var reader = new FileReader();
reader.onload = function (e) {
var data = e.target.result;
/*Converts the excel data in to object*/
if (xlsxflag) {
var workbook = XLSX.read(data, { type: 'binary' });
}
else {
var workbook = XLS.read(data, { type: 'binary' });
}
/*Gets all the sheetnames of excel in to a variable*/
var sheet_name_list = workbook.SheetNames;
console.log(sheet_name_list);
var cnt = 0; /*This is used for restricting the script to consider only first
sheet of excel*/
sheet_name_list.forEach(function (y) { /*Iterate through all sheets*/
/*Convert the cell value to Json*/
if (xlsxflag) {
var exceljson = XLSX.utils.sheet_to_json(workbook.Sheets[y]);
}
else {
var exceljson = XLS.utils.sheet_to_row_object_array(workbook.Sheets[y]);
}
//Download & View Subscriptions
if (exceljson.length > 0 && cnt == 1) {
metadata = [];
fetch("metadata.json")
.then(response => response.json())
.then(json => {
metadata = json;
console.log(metadata);
user_metadata1 = [], obj_m_processed = [];
for (var i in exceljson) {
var obj = { email: exceljson[i].email, name: exceljson[i].team_alias, id: exceljson[i].autodesk_id };
for (var j in metadata) {
if (exceljson[i].email == metadata[j].email) {
obj.GEO = metadata[j].GEO;
obj.COUNTRY = metadata[j].COUNTRY;
obj.CITY = metadata[j].CITY;
obj.PROJECT = metadata[j].PROJECT;
obj.DEPARTMENT = metadata[j].DEPARTMENT;
obj.CC=metadata[j].CC;
obj_m_processed[metadata[j].email] = true;
}
}
obj.GEO = obj.GEO || '-';
obj.COUNTRY = obj.COUNTRY || '-';
obj.CITY = obj.CITY || '-';
obj.PROJECT = obj.PROJECT || '-';
obj.DEPARTMENT = obj.DEPARTMENT || '-';
obj.CC = obj.CC || '-';
user_metadata1.push(obj);
}
for (var j in metadata) {
if (typeof obj_m_processed[metadata[j].email] == 'undefined') {
user_metadata1.push({ email: metadata[j].email, name: metadata[j].name, id: metadata[j].autodesk_id,
GEO: metadata[j].GEO,
COUNTRY : metadata[j].COUNTRY,
CITY : metadata[j].CITY,
PROJECT : metadata[j].PROJECT,
DEPARTMENT : metadata[j].DEPARTMENT,
CC:metadata[j].CC
});
}
}
document.getElementById("headings4").innerHTML = "MetaData Mapping";
BindTable(user_metadata1, '#user_metadata1
cnt++;
});
$('#exceltable').show();
}
if (xlsxflag) {/*If excel file is .xlsx extension than creates a Array Buffer from excel*/
reader.readAsArrayBuffer($("#excelfile")[0].files[0]);
}
else {
reader.readAsBinaryString($("#excelfile")[0].files[0]);
}
}
else {
alert("Sorry! Your browser does not support HTML5!");
}
}
else {
alert("Please upload a valid Excel file!");
}
}
Here is how the json is bind after mapping metadata
function BindTable(jsondata, tableid) {/*Function used to convert the JSON array to Html Table*/
var columns = BindTableHeader(jsondata, tableid); /*Gets all the column headings of Excel*/
for (var i = 0; i < jsondata.length; i++) {
var row$ = $('<tr/>');
for (var colIndex = 0; colIndex < columns.length; colIndex++) {
var cellValue = jsondata[i][columns[colIndex]];
if (cellValue == null)
cellValue = "";
row$.append($('<td/>').html(cellValue));
}
$(tableid).append(row$);
}
}
function BindTableHeader(jsondata, tableid) {/*Function used to get all column names from JSON and bind the html table header*/
var columnSet = [];
var headerTr$ = $('<tr/>');
for (var i = 0; i < jsondata.length; i++) {
var rowHash = jsondata[i];
for (var key in rowHash) {
if (rowHash.hasOwnProperty(key)) {
if ($.inArray(key, columnSet) == -1) {/*Adding each unique column names to a variable array*/
columnSet.push(key);
headerTr$.append($('<th/>').html(key));
}
}
}
}
$(tableid).append(headerTr$);
return columnSet;
}

"Debugging connection was closed: Render process was gone", when trying to download a 7gb from cdn

We are trying to download a 7 GB from CDN using JSZip.js. The chrome browser suddenly seems to close the connection when the download reaches 3.5gb every time. The approximate time is around 15 mins. Is there a way we increase the tolerant time to 1 hr say?
$("#downloadJSZip").on('click', function () {
var result = [{ "cdn": "url....", "filename": "7.84 gb.zip", "size": 4194304, "path": "7.84 gb" }];
var Promise = window.Promise;
if (!Promise) {
Promise = JSZip.external.Promise;
}
function urlToPromise(url) {
return new Promise(function(resolve, reject) {
JSZipUtils.getBinaryContent(url, function (err, data) {
if(err) {
reject(err);
} else {
resolve(data);
}
});
});
}
var fileNameArray = [];
function changeFileName(fileName,j){
var i = fileName.lastIndexOf('.');
var newfilename = fileName.slice(0,i)+"--"+j+fileName.slice(i);
if(fileNameArray.indexOf(newfilename) != -1){
j = j+1;
changeFileName(fileName,j);
}
return newfilename;
}
var zip = new JSZip();
// find every checked item
result.forEach(function(file){
var filename = file.filename;
if(fileNameArray.indexOf(filename) != -1){
var newfilename = changeFileName(filename,1);
filename = newfilename;
}
fileNameArray.push(filename);
var url = file.cdn;
var folder = (file.path);
zip.folder(folder).file(filename, urlToPromise(url), {binary:true});
// zip.file(filename, urlToPromise(url), {binary:true});
});
// when everything has been downloaded, we can trigger the dl
zip.generateAsync({type:"blob",
}, function updateCallback(metadata) {
var msg = "progression : " + metadata.percent.toFixed(2) + " %";
if(metadata.currentFile) {
msg += ", current file = " + metadata.currentFile;
}
console.log(msg);
console.log(metadata.percent|0);
})
.then(function callback(blob) {
// see FileSaver.js
//console.log("blob=====>");
//console.dir(blob);
//saveAs(blob, "example.zip") ;
saveAs(blob, $scope.folderName+".zip") ;
//console.log("done !");
}, function (e) {
});
});
Is this chrome browser configuration?

No output after trying loading data on the fly using Json and ajax

var pageCounter = 1;
var animalContainer = document.getElementById("animal-info");
var btn = document.getElementById("btn");
btn.addEventListener("click", function() {
var ourRequest = new XMLHttpRequest();
ourRequest.open('GET', 'https://learnwebcode.github.io/json-example/animals-1.json');
ourRequest.onload = function() {
if (ourRequest.status >= 200 && ourRequest.status < 400) {
var ourData = ourRequest.responseText;
renderHTML(ourData);
} else {
console.log("We connected to the server, but it returned an error.");
}
};
ourRequest.onerror = function() {
console.log("Connection error");
};
ourRequest.send();
pageCounter++;
if (pageCounter > 3) {
btn.classList.add("hide-me");
}
});
function renderHTML(data) {
var htmlString = "";
for (i = 0; i < data.length; i++) {
htmlString += "<p>" + data[i].name + " is a " + data[i].species + "</p>";
}
animalContainer.insertAdjacentHTML('beforeend', htmlString);
}
I have checked every thing and i don't know what is wrong with the code... I am trying to load information on the fly using Json and ajax
You did not parse your data to JSON so what you have to do is parse your "ourData" to JSON.. Remove and add this line of to your code.
var ourData = JSON.parse(ourRequest.responseText);

Sending multiple JSON request to server under one request nodejs

I have some files downloaded and I wanna push them to a server. Every time I try to push the file I get the error
Error: Can't set headers after they are sent.
I have a for loop that reads all the files and parse them and after that I want to send them to the server one by one.
app.get('/dataparser', function(req, res) {
var fs = require('fs');
var obj;
var jsonGis = new Array();
var jsonArray;
var filePaths = [];
const downloadFolder = './sampletest/';
var mtimes = {};
var reloadTimes = 10000;
fs.readdir(downloadFolder, (err, files) => {
files.forEach(file => {
filePaths.push("sampletest/" + file);
});
})
var execFunction = function() {
for (var i = 0; i < filePaths.length; i++) {
parseFile(filePaths[i], mtimes[filePaths[i]]);
}
};
execFunction();
setInterval(execFunction, reloadTimes);
function parseFile(fileName, lastModifiedTime) {
fs.stat(fileName, function(err, fd) {
for (var i=0, len = filePaths.length; i<len; i++) {
if (fd.mtime !== lastModifiedTime) {
mtimes[fileName] = fd.mtime;
fs.readFile(filePaths[i], function(err, data) {
if (err) {
return console.error(err);
}
obj = JSON.parse(data);
jsonGis.push('"Person1"');
jsonGis.push('"' + obj.pages[1].answers[2].values[0] + '"');
jsonGis.push('"person2"');
jsonGis.push('"' + obj.pages[1].answers[0].values[0] + '"');
jsonGis.push('"codewals"');
jsonGis.push('"42343GSDS"');
jsonGis.push('"geometry":{');
jsonGis.push('"x":' + obj.pages[1].answers[4].values[0].coordinates.latitude + ',');
jsonGis.push('"y":' + obj.pages[1].answers[4].values[0].coordinates.longitude);
var str = "[{ " + jsonGis[0] + jsonGis[1] + jsonGis[2] + ": " + jsonGis[3] + "," + jsonGis[4] + ": " + jsonGis[16] + "}}]"
//pushing to the server
console.log("check here");
console.log(str);
var qs = require("querystring");
var http = require("http");
var options = {
"method": "POST",
"hostname": "twst2.gtw.com",
"port": null,
"path": "localpath/",
"headers": {
"accept": "application/json",
"content-type": "application/x-www-form-urlencoded",
}
};
var req = http.request(options, function(res) {
var chunks = [];
res.on("data", function(chunk) {
chunks.push(chunk);
});
res.on("end", function() {
var body = Buffer.concat(chunks);
console.log(body.toString());
});
});
req.write(qs.stringify({features: str}));
res.send("The server was updated");
req.end();
});
}
}
});
};
Just need to send the data to the server from file1.json and then file2.json then file3.json and so on.
Try creating a separate callback function that includes .write, .send, and .end, then let it be the callback for .request.

How convert tsv to Json

I want to make a dynamic graph based on a json file. I have seen many examples with tsv but I donot how to convert it to json.
That is the part that I want to change from tsv to json but I donot know how!
d3.tsv("data/data.tsv", function(error, data) {
data.forEach(function(d) {
d.date = parseDate(d.date);
d.close = +d.close;
});
when I use
d3.json("data/data.json", function(data) {
data.forEach(function d) {
d.date = parseDate(d.date);
d.close = +d.close;
}
});
it gives this error: Uncaught type error: cannot call method 'forEach' of undefined!
Thanks for your suggestions :)
try to do something like this
d3.json("data/data.json", function(data) {
data.forEach(function d) {
d.date = parseDate(d.date);
d.close = +d.close;
}
});
d3.js have support for json, https://github.com/mbostock/d3/wiki/Requests
The syntax around your forEach is a little off; try this instead:
d3.json("data/data.json", function(data) {
data.forEach(function(d) {
d.date = parseDate(d.date);
d.close = +d.close;
});
});
(As Felix points out, this will only work if your JSON object is defined and is an array)
Here a small code where you'll be able to convert tsv to json. It could help you...
ps : here is typescript, but you can easily convert it to vanilla javascript ;)
// Set bunch of datas into format object
tsvToJson(datas: string): Array<Object>{
// Separate each lines
let array_datas = datas.split(/\r\n|\r|\n/g);
// Separate each values into each lines
var detailed_datas = [];
for(var i = 0; i < array_datas.length; i++){
detailed_datas.push(array_datas[i].split("\t"));
}
// Create index
var index = [];
var last_index = ""; // If the index we're reading is equal to "", it mean it might be an array so we take the last index
for(var i = 0; i < detailed_datas[0].length; i++){
if(detailed_datas[0][i] == "") index.push(last_index);
else {
index.push(detailed_datas[0][i]);
last_index = detailed_datas[0][i];
}
}
// Separate data from index
detailed_datas.splice(0, 1);
// Format data
var formated_datas = [];
for(var i = 0; i < detailed_datas.length; i++){
var row = {};
for(var j = 0; j < detailed_datas[i].length; j++){
// Check if value is empty
if(detailed_datas[i][j] != ""){
if(typeof row[index[j]] == "object"){
// it's already set as an array
row[index[j]].push(detailed_datas[i][j]);
} else if(row[index[j]] != undefined){
// Already have a value, so it might be an array
row[index[j]] = [row[index[j]], detailed_datas[i][j]];
} else {
// It's empty for now, so let's say first that it's a string
row[index[j]] = detailed_datas[i][j];
}
}
}
formated_datas.push(row);
}
console.log(formated_datas); // #TODO : remove this
return formated_datas;
}
I transpile and resume Wetteren's code:
convertTSVtoJSON(tsvData) {
const formattedData = tsvData.split(/\r\n|\r|\n/g).filter(e => !!e).map((parsedEntry) => parsedEntry.split("\t"));
const tsvHeaders = formattedData.shift();
return formattedData.map(formattedEntry => {
{
return tsvHeaders.reduce((jsonObject, heading, position) => {
jsonObject[heading] = formattedEntry[position];
return jsonObject;
}, {});
}
});
}