NODEJS node-mysql cannot pass query results in nested queries - mysql

I am using node, angular and mysql, the node routes would return a json that would be processed by angular, the json is returned by first querying the mysql DB using the node-mysql module,
In the below code I am unable to set the value of CreatedID, but the value gets logged properly in terminal. I was facing the same issue in the 1st query but then sorted it in the below code, now unable to access the nested query results.
var mysql = require('node-mysql/node_modules/mysql');
var connection = mysql.createConnection({
host : 'localhost',
user : "root",
password: "",
database:'designtaskmanager'
});
connection.connect();
var allDbCalls = function() {
var sendData = {};
var rowData = {};
var temp={};
var _this = this;
this.sendTask = function(callback) {
module.exports.taskData = rowData;
callback['success']();
};
this.getTask = function(callback) {
var strQuery = "select * from task";
connection.query( strQuery, function(err, rows){
if(err)
{
callback['failure']();
throw err;
}
else
{
//rowData = rows;
var tasks=[];
for (var i in rows)
{
var Title = rows[i].task_title;
var TaskDescription=rows[i].task_description;
var TaskCategory=rows[i].task_category;
var TaskID=rows[i].task_id;
var TaskStatus=rows[i].task_status;
var TaskStatusMessage
var CreatedBy;
var TaskCreationDate=rows[i].task_creation_date;
var _MS_PER_DAY = 1000 * 60 * 60 * 24;
var currentdate = new Date();
var ddd=dateDiffInDays(TaskCreationDate,currentdate);
function dateDiffInDays(a, b) {
// Discard the time and time-zone information.
var utc1 = Date.UTC(a.getFullYear(), a.getMonth(), a.getDate());
var utc2 = Date.UTC(b.getFullYear(), b.getMonth(), b.getDate());
return Math.floor((utc2 - utc1) / _MS_PER_DAY);
}
if(TaskStatus==0)
{
TaskStatus="label-info";
TaskStatusMessage="Ongoing since";
}
else if(TaskStatus==1)
{
TaskStatus="label-default";
TaskStatusMessage="Paused since"
}
else if(TaskStatus==2)
{
TaskStatus="label-success";
TaskStatusMessage="Completed in"
}
//USER DETAILS QUERY
var crid=rows[i].task_created_by;
var creatorQuery = "select user_email from users where user_id like ?";
connection.query( creatorQuery,[crid], function(err, createdbyrows){
if(err)
{
callback['failure']();
throw err;
}
else
{
for(var j=0; j< createdbyrows.length;j++)
{
CreatedBy=createdbyrows[0].user_email;
console.log(j);
}
console.log(CreatedBy);
}
});
var taskItem={"TaskID":TaskID,"TaskTitle":Title,"TaskDescription":TaskDescription,"TaskCategory":TaskCategory,"CreatedBy":CreatedBy,"TaskStatus":TaskStatus,"TaskStatusMessage":TaskStatusMessage,"DifferenceInDays":ddd};
tasks.push(taskItem);
}
rowData=tasks;
_this.sendTask(callback);
}
});
}
}
module.exports = function () {
var instance = new allDbCalls();
return instance;
};

The reason that you're seeing it on the console but not in the callback is due to a misunderstanding of asynchronous programming. When you:
for(var i in rows) {}
You are actually queuing up all of those queries at the same time, then, immediately after you try to set rowData to an empty array:
rowData=tasks; // remember, none of the queries have finished yet
_this.sendTask(callback);
So you pretty much call your callback when tasks is still an empty array. Remember, you can't call your final callback until ALL of your nested queries have finished!
To accomplish this, you may want to look at the async library: https://github.com/caolan/async#eachSeries
This will help you accomplish what you really want.
var async = require("async");
async.eachSeries(rows, function(row, cb) {
// Do each query here
// then call cb() when done, which tells the async library
// to "go to the next item in the array"
}, function(err) {
// This will get called when all of the single queries are finished
// Check err, then call your callback
_this.sendTask(callback);
});

Related

Loopback unable to create and do operations on related models in model JS file

Unable to create relation in model.js file while through model.json file, it's working fine. I want to update booking table field on any updation in bidding table but my relation is not working in model.js file.
Bidding.belongsTo(Booking, {foreignKey: 'bookingId'});
^
ReferenceError: Booking is not defined
{
"error": {
"statusCode": 500,
"name": "TypeError",
"message": "Bidding.booking is not a function",
"stack": "TypeError: Bidding.booking is not a function\n.."
}
}
'use strict';
//var loopback = require('loopback');
//var boot = require('loopback-boot');
//var app = module.exports = loopback();
module.exports = function(Bidding) {
// var app = require('../../server/server');
// var Booking = app.models.Booking;
//Bidding.belongsTo(Myuser, {foreignKey: 'driver_id'});
Bidding.belongsTo(Booking, {foreignKey: 'bookingId'});
Bidding.observe('before save', function beforeSave(ctx, next) {
if (ctx.instance) {
//on create
ctx.instance.created = new Date();
ctx.instance.modified = new Date();
} else {
// on edit
// ctx.instance.lastUpdated = new Date();
console.log('updatesdd');
//Bidding.Booking.upsertWithWhere({id: ctx.instance.id},{ 'username': username}, function(err, results) {});
Bidding.booking(function(err, booking) {
console.log(ctx.booking);
});
}
next();
});
Bidding.observe('loaded', function beforeaccess(ctx, next) {
console.log(ctx.data);
next();
});
};
By following way you can perform the operations on your relates tables. I go through inbuilt user.js file where they have relation with AccessToken table and find the following solution for my problem.
Query which load you related table -
Here "booking" model has belongs to relation define in bidding.json
var booking = ctx.Model.relations.booking.modelTo;
Update Query call through StrongLoop API Explorer:
http://0.0.0.0/api/Biddings/update?where=%7B%22id%22%3A%2211%22%2C%22bookingId%22%3A6%7D
Where: {"id":"11","bookingId":6}
data: {"bid_status":"Accept"}
Bidding.observe('before save', function beforeSave(ctx, next) {
if (ctx.instance) {
//on create
ctx.instance.created = new Date();
ctx.instance.modified = new Date();
} else {
//on update
if(ctx.data.bid_status == "Accept");
{
if (ctx.where && ctx.where.bookingId && ctx.where.id) {
var bookingId = ctx.where.bookingId;
var booking = ctx.Model.relations.booking.modelTo;
booking.upsertWithWhere({id: bookingId},{ 'booking_status': 'Confirmed'}, function(err, results) {});
}
}
}
next();
});

want to convert my webapp to desktop app with data persistence

I have created a payroll webapp and have used mysql and node js to build it.
The problem is that I am not sure how to convert it to a desktop webapp.
I have used the node-mysql module for this, so do I need to change something now
or just convert it using node webkit?
How does this work, really?
Here's what my server.js looks like now. so , do I need to make any changes to the code again?
var express = require('express');
var path = require('path');
var bodyParser = require('body-parser');
var app = express();
var mysql = require('mysql');
//var ejsLint=require('./server.js');
//ejsLint.lint('attendance-data.ejs', '-p');
var connection = mysql.createConnection({
host:'localhost',
user:'root',
password:'',
database:'employees',
multipleStatements:true
});
app.set('view engine','ejs');
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.get('/', function (req,res){
res.render('index');
});
var daValue=227.30;
app.get('/da',function(req,res){
res.render('da');
});
app.post('/da-data',function(req,res){
daValue = Number(req.body.da);
var data = {da :daValue};
console.log('da is :',daValue);
res.render('da-data.ejs',data);
});
app.get('/add-employee',function(req,res){
res.render('add-employee');
});
app.post('/add-employee',function(req,res){
res.status(200).send();
console.log('employee '+req.body.name + ' added');
});
connection.connect(function(err){
if(err){
console.log('Error connecting to Db');
return;
}
console.log('Connection established');
});
app.post('/form',function (req,res){
//employee details from add-employee.html page
var name=req.body.name;
var designation = req.body.designation;
var pan = req.body.pan;
var aadhar = req.body.aadhar;
var bank = req.body.bank;
var basicSalary = req.body.salary;
var allowance = req.body.allowance;
var grossSalary = req.body.salarygross;
var esi = req.body.esi;
var uan = req.body.uan;
var details = {name:name,designation:designation,pan:pan,aadhar:aadhar,
bank:bank,basic_salary:basicSalary,other_allowances:allowance,gross_salary:grossSalary,esi:esi,uan:uan};
// for sending data as objects into database
// use {name of database column : name of the variable where it's value is stored}
// example {wages_rate is the name of database column : wagesRate is the variable where
// value is stored}
var query = connection.query('INSERT INTO employee_details SET ?',details,function(err,result){
if(err){
console.log(err);
}
console.log(query.sql);
});
res.status(200).send('employee ' + name + 'with salary of '+salary+ ' added');
});
app.get('/show',function (req,res){
connection.query('SELECT * FROM employee_details',function(err,rows){
if(err) throw err;
console.log('Data received');
console.log(rows);
var data ="Id "+"<strong>" +" name "+" designation "+" uan"+"</strong>";
for(var i=0;i<rows.length;i++){
data = data+"<br>" + (i+1)+". "+rows[i].name +" "+rows[i].designation+" "+rows[i].uan+"<br>";
}
res.send(data);
});
});
var rowsLength;
var salaryFromRow;
var salaryArr=[];
var allowanceFromRow;
var allowanceArr=[];
var designationArr=[];
app.get('/attendance',function (req,res){
connection.query('SELECT * FROM employee_details',function(err,rows){
if(err) {
throw err;
}else{
rowsLength = rows.length;
for(var i=0;i<rowsLength;i++){
salaryFromRow = rows[i].salary;
salaryArr.push(salaryFromRow);
allowanceFromRow = rows[i].allowance;
allowanceArr.push(allowanceFromRow);
designationArr.push(rows[i].designation);
console.log('designation is ',designationArr);
}
res.render('attendance',{rows:rows});
}
});
});
app.post('/attendance-data',function(req,res){
var uanFromHtml;
var nameFromHtml;
var daysPresent;
var attendance;
var nameForm;
var designation;
var monthFromHTML;
var t;
var realBasicSalary;
var realOtherAllowances;
var realGrossSalary;
var pfBasic;
var esiGross;
var pTax=0;
var netAmount;
var advance=0;
var finalData = [];
for(var i=1;i<=rowsLength;i++){
var dataArr = [];
var finalTable = [];
attendance = "attendance"+i;
t=i-1;
salaryForPerson = salaryArr[i];
allowanceForPerson = allowanceArr[i];
console.log('req.body is ', req.body);
daysPresent = Number(req.body[attendance]);
nameFromHtml = req.body.name[t];
var nameArr = req.body.name;
uanFromHtml = req.body.uan[t];
monthFromHTML = req.body.monthyear;
var uanArr = req.body.uan;
realBasicSalary = Math.ceil((req.body.basicsalary[t])*(daysPresent/30)) ;
realOtherAllowances = Math.ceil((req.body.otherallowance[t])*(daysPresent/30));
realGrossSalary = Math.ceil((req.body.grosssalary[t])*(daysPresent/30));
console.log('realBasicSalary is '+realBasicSalary+' and realGrossSalary is '+realGrossSalary+' and realOtherAllowances is '+ realOtherAllowances);
pfBasic = Math.ceil((12/100)*realBasicSalary);
esiGross = Math.ceil((1.75/100)*realGrossSalary);
if(realBasicSalary>10000 && realBasicSalary<=15000){
pTax = 110;
}else if(realBasicSalary>=15001 && realBasicSalary<=25000){
pTax = 130;
}else if(realBasicSalary>=25001 && realBasicSalary<=40000){
pTax = 150;
}else if(realBasicSalary>=40001){
pTax = 200;
}
netAmount = realGrossSalary - (pTax + pfBasic + esiGross + advance);
console.log('realGrossSalary is '+realGrossSalary + ' and realBasicSalary is '+realBasicSalary+
'pTax is '+ pTax+ 'pfBasic is '+pfBasic +'esiGross is '+esiGross);
console.log('namefromhtml is : ', nameFromHtml);
console.log('attendance is :',attendance);
console.log('days present is :',daysPresent);
console.log('monthyear is : ',monthFromHTML);
dataArr.push(monthFromHTML,uanFromHtml,nameFromHtml,daysPresent,realBasicSalary,realOtherAllowances,realGrossSalary,pTax);
finalData.push(dataArr);
/* uanArr.push(uanFromHtml);
nameArr.push(nameFromHtml);
daysArray.push(daysPresent);
*/
console.log('dataArr is : ',dataArr);
console.log('finalData is : ',finalData);
}
var attendanceData = {monthyear :monthFromHTML,rows:rowsLength,uanarr:uanArr,designationarr:designationArr,
namearr:nameArr,finaldata:finalData,realbasicsalary:realBasicSalary,realgrosssalary:realGrossSalary,ptax:pTax,advance:advance};
connection.query("INSERT INTO attendance_details(month_year,uan,name,days_present,real_basic_salary,other_allowances,gross_salary,ptax) VALUES ?",
[finalData], function(err) {
if (err){
var errors = err;
console.log(errors);
res.send(errors);
}else{
//put database query for inserting values here
res.render('attendance-data.ejs', attendanceData);
}
});
});
/*
app.get('/final',function(req,res){
connection.query('SELECT name,designation,salary,wages_rate FROM employee_details;SELECT uan,da,days_present,total_wages FROM attendance_details;',function(err,rows){
if(err){
console.error('MySQL — Error connecting: ' + err.stack);
}else{
var rowsNumber = rows.length;
console.log('rows is :',rows);
var nameFinal;
var designationFinal;
var salaryFinal;
var wagesrateFinal;
var uanFinal;
var daFinal;
var daysFinal;
var totalwagesFinal;
var nameFinalarr = [];
var designationFinalarr =[];
var salaryFinalarr = [];
var wagesrateFinalarr =[];
var uanFinalarr =[];
var daFinalarr =[];
var daysFinalarr = [];
var totalwagesFinalarr =[];
for(var i=0;i<rowsNumber;i++){
nameFinalarr.push(rows[i].name);
designationFinalarr.push(rows[i].designation);
salaryFinalarr.push(rows[i].salary);
wagesrateFinalarr.push(rows[i].wages_rate);
uanFinalarr.push(rows[i].uan);
daysFinalarr.push(rows[i].da);
daysFinalarr.push(rows[i].days_present);
totalwagesFinalarr.push(rows[i].total_wages);
}
console.log('nameFinalarr is :', nameFinalarr);
console.log('daysFinalarr is :', daysFinalarr);
}
res.render('final',{rows:rowsNumber,name:nameFinal,designation:designationFinal,salary:salaryFinal,wagesrate:wagesrateFinal,uan:uanFinal,da:daFinal,
days:daysFinal,
totalwages:totalwagesFinal});
});
});
*/
app.get('/select-month',function(req,res){
connection.query('SELECT month_year,name FROM attendance_details',function(err,rows){
if(err){
throw err;
}else{
var rowsLength = rows.length;
console.log('rows is ',rows);
res.render('select-month.ejs',{rows:rows});
}
});
});
app.get('/salary-sheet',function(req,res){
var month = req.query.selectpicker;
var employeeName = req.query.selectpicker2;
console.log('employeeName is ',employeeName);
connection.query('SELECT * FROM attendance_details WHERE month_year='+"'"+month+"' AND name='"+employeeName+"'",function(err,rows){
if(err){
throw err;
}else{
var rowsLength = rows.length;
console.log('rows is ',rows);
var uanarr=[];
var namearr=[];
var daysarr=[];
var realBasicSalaryarr=[];
var realOtherAllowancesarr=[];
var grossSalaryarr=[];
var ptaxarr=[];
var advance=0;
for(var i=0;i<rowsLength;i++){
uanarr.push(rows[i].uan);
namearr.push(rows[i].name);
daysarr.push(rows[i].days_present);
realBasicSalaryarr.push(rows[i].real_basic_salary);
realOtherAllowancesarr.push(rows[i].other_allowances);
grossSalaryarr.push(rows[i].gross_salary);
ptaxarr.push(rows[i].ptax);
}
console.log('realBasicSalaryarr is ',realBasicSalaryarr);
console.log('namearr is ',namearr);
res.render('salary-sheet.ejs',{advance:advance,rows:rows,monthyear:month,uanarr:uanarr,namearr:namearr,daysarr:daysarr,basicsalaryarr:realBasicSalaryarr,
realotherallowancesarr:realOtherAllowancesarr,realgrosssalaryarr:grossSalaryarr,ptaxarr:ptaxarr});
}
});
});
app.get('/add-company',function(req,res){
res.render('add-company.ejs');
});
app.get('/style.css',function(req,res){
res.sendFile(path.join(__dirname,'/style.css'));
});
app.get('/main.js',function(req,res){
res.sendFile(path.join(__dirname,'/main.js'));
});
app.get('/cmain.js',function(req,res){
res.sendFile(path.join(__dirname,'/cmain.js'));
});
var port=8080;
app.listen(8080,function(req,res){
console.log(`Payroll app listening on port ${port}!` );
});
Could you tell me how to convert this to a desktop app?
In order to make this work offline, in other words, stand-alone, you'll need to revisit your approach regarding persistence. MySQL is not something I'd attempt to bundle with the application. You would ideally use something from the link Yonghoon Lee suggested, as you need the app to have an embedded database instead of an external one.
I've had success doing something similar, although switching to IndexDB meant I had to rewrite all my queries as it's a NoSQL database. Please resist using Web SQL database, it's not standard and could disappear at any time.
Other than that, I don't see anything that's an obvious problem to convert to a NWJS application. You can refer to the official documentation for some examples of how to get started.
It cannot easily be ported. You should not expect the people on stackoverflow to do all the work for you. It is not a simple task but i will try to guide you trough it.
1.) understand what NWJS is. NWJS ist not just a node.js server but also a client. That means you can use both the code of node.js and javascript code that is ment to run on the client.
2.) You server js is not necessary. The entry point of you application is no server js file but an index.html file.
3.) You js code should be added into the tag of the index html. Her you can create a main.js file with the code you want to run and add it as a script tag.
4.) Install mysql into your nwjs package with npm install mysql --save
You have to run this command in the folder that has you package.json
5.) If you want to use routes you have to install express.js
The routes will be localhost routes on your computer
Now you have to consider what you really want to do? From where should the routes be accessible? Would you want an app where you have buttons that you can click and then provide data to the mysql database?
If you really have the interest to port it to nwjs you have to understand how nwjs works. I have given you the first steps but without further information I cannot give you more advises.
Wish you the best of luck. If you understand the structure of nwjs it will be easy to port it ;)

Call sql query synchrously

Currently working on node rest api project where I want to fetch data for a list of data. for example : I have a list of post_id([1,2,3....]) for a particular tag(mobile) and for each post_id I want to retrieve post title and description from mysql database. But calling sql query is synchrounous.
How to control flow for each post id result to combine in one.
my db calling code is here :
var express = require('express');
var app = express();
var bodyParser = require('body-parser'); // call body-parser
var addData = require('./dbhandler/addData'); // call database handler to insertdata
var getData = require('./dbhandler/getData');
//route function to get feeds by tags
router.route('/postfeedsbytags/:tag')
// get all new article feeds filtered by tag
.get(function(req,res){
var success;
console.log(req.params.tag)
var json_results = [];
getData.getPostFeedsByTag(req.params.tag,function(error, results, fields){
if (!error){
for (var i = 0; i < results.length; i++) {
getData.getPostFeedsByPostId(results[0]['post_id'],function(error, results, fields){
if (!error){
success = 1;
json_results.push(results[0]);
res.json({"success" : success, "datasets" : json_results});
} else{
success = 0;
console.log('Error while performing Query.'+error);
res.json({"success" : success});
}
});
}
// res.json({"success" : success, "datasets" : results});
} else{
success = 0;
console.log('Error while performing Query.'+error);
res.json({"success" : success});
}
});
});
I think you can use the IN operator in the query to get all the posts in a single query and then iterate over it.
If you don't want to use IN operator then use async library for flow control. You can use the async.map function from it.

Node.js asynchronous nested mysql queries

To set the scenario for the code, the database stores Documents, and each document has the potential to have Images associated with them.
I have been trying to write a route that queries the database for each Document that has Images related to them, storing this data in JSON which is returned to the ajax request when completed, so the data can be viewed on the page. The closest I have got so far is the below attempt (see code).
router.post('/advanced_image_search', userAuthenticated, function(req, res, next) {
async.waterfall([
// First function is to get each document which has an image related
function getDocuments(callback){
connection.query(`SELECT DISTINCT(Document.document_id), Document.doc_name, Document.doc_version_no, Document.doc_date_added
FROM Document WHERE doc_type = 'image'`, function(err, results) {
if (err) {
callback(err, null);
return;
}
// The Object containing the array where the data from the db needs to be stored
var documents = {
'docs': []
};
// foreach to iterate through each result found from the first db query (getDocuments)
results.forEach(function(result) {
// New object to store each document
var document = {};
document.entry = result;
// This is the array where each image assciated with a document will be stored
document.entry.images = [];
// Push each document to the array (above)
documents.docs.push(document);
var doc_id = result.document_id;
})
// Returning the results as 'documents' to the next function
callback(null, documents);
})
},
function getImages(documents, callback){
// Variable assignement to the array of documents
var doc_array = documents.docs;
// Foreach of the objects within document array
async.forEachOf(doc_array, function(doc, key, callback){
// Foreach object do the following series of functions
async.waterfall([
function handleImages(callback){
// The id of the document to get the images for
var doc_id = doc.entry.document_id;
connection.query(`SELECT * FROM Image, Document WHERE Image.document_id = '${doc_id}' AND Image.document_id = Document.document_id`, function(err, rows) {
if (err) {
callback(err, null);
return;
}
callback(null, rows);
})
},
// Function below to push each image to the document.entry.images array
//
function pushImages(rows, callback){
// If multiple images are found for that document, the loop iterates through each pushing to the images array
for (var j = 0; j < rows.length; j++) {
// Creating new object for each image found so the data can be stored within this object, then pushed into the images array
var image = {
'image_name': rows[j].image_name
};
doc.entry.images.push(image);
}
callback(null, doc_array);
}
], function(err, doc_array){
if (err) {
console.log('Error in second waterfall callback:')
callback(err);
return;
}
console.log(doc.entry);
// callback(null, doc_array);
})
}, function(err, doc_array){
if (err) {
callback(err);
return;
}
callback(null, doc_array);
});
callback(null, doc_array);
}
], function(err, doc_array) {
if (err){
console.log('Error is: '+err);
return;
}
// The response that should return each document with each related image in the JSON
res.send(doc_array);
})
});
At the moment the results returned are:
1:
{entry: {document_id: 1, doc_name: "DocumentNameHere", doc_version_no: 1,…}}
entry:
{document_id: 1, doc_name: "DocumentNameHere", doc_version_no: 1,…}
doc_date_added:"2016-10-24"
doc_name:"DocumentNameHere"
doc_version_no:1
document_id:1
images:[]
As can be seen above, the images array remains empty even though with testing, the images are being found (console.log).
I hope someone is able to assist with this, as I am struggling to find the problem with this complex one.
Thanks
There are several async operations going on here and each operation needs a callback. See revised code:
router.post('/advanced_image_search', userAuthenticated, function(req, res, next) {
var getDocuments = function(next) {
// Function for getting documents from DB
var query = `SELECT DISTINCT(Document.document_id), Document.doc_name, Document.doc_version_no, Document.doc_date_added FROM Document WHERE doc_type = 'image'`; // Set the query
connection.query(query, function(err, results) {
// Run the query async
if(err) {
// If err end execution
next(err, null);
return;
}
var documentList = []; // Array for holding docs
for(var i=0; i<results.length; i++) {
// Loop over results, construct the document and push to an array
var documentEntry = results[i];
var documentObject = {};
documentObject.entry = documentEntry;
documentObject.entry.images = [];
documentObject.id = documentEntry.document_id;
documentList.push(documentObject);
}
next(null, documents); // Pass to next async operation
});
};
var getImages = function(documents, next) {
// Function for getting images from documents
var finalDocs = []; // Blank arry for final documents with images
for (var i=0; i<documents.length; i++) {
// Loop over each document and construct the query
var id = documents[i].id;
var query = `SELECT * FROM Image, Document WHERE Image.document_id = '${doc_id}' AND Image.document_id = Document.document_id`;
connection.query(query, function(err, images) {
// Execute the query async
if(err) {
// Throw error to callback
next(err, null);
return;
}
var processedDoc = processImages(documents[i], images); // Call a helper function to process all images into the document object
finalDocs.push(processedDoc); // Push the processed doc
if(i === documents.length) {
// If there are no more documents move onto next async
next(null, finalDocs);
}
});
}
};
var processImages = function(doc, images) {
for (var i=0; i< images.length; i++) {
// Loop over each document image - construct object
var image = {
'image_name': rows[j].image_name
};
doc.entry.images.push(image); // Push image into document object
}
return doc; // Return processed doc
};
getDocuments(function(err, docs) {
if(err) {
// Your error handler
}
if(docs) {
getImages(docs, function(err, finalDocs) {
if(err) {
// Your error handler
}
if(finalDocs) {
console.log(finalDocs);
res.status(200).json(finalDocs); // Send response
}
});
}
});
});
First we create a function to get documents - This function accepts a callback as its parameter. We run our query and construct our doc list. Then we return the document list by executing our callback
Next we run a function to get our images for each document. This function accepts our document list and a callback as our parameters. It retrieves images for each document and calls a helper function (sync)
Our helper function processes the images into each document and returns the processed document.
We then finish our operations by returning an array of processed documents via the second callback.
Other notes
We could tide this up by structuring this procedural style code into a contained JSON object
The nesting of the function executions at the end of the document could be cleaned up further
I've avoided using the async libray as it helps better understanding of the callback model
Event emitters could be used to flatten the callbacks - see https://nodejs.org/dist/latest-v7.x/docs/api/events.html
Hope this helps
Dylan

Creating synchronous queries with node-mysql

I'm trying to ensure that one mysql query leads to another and is not completed until all of its children queries are completed. So for example, I start with one select and stream rows and execute subsequent queries from that row result. This is doable with callbacks, but I end up running out of memory, so I'd like to slow down the process and run batches, but due to the async nature of the dispatch, I can't keep things in phase and end the connection after all the rows have been processed.
Here's an example:
var query = conn.query('select id from table1 limit 10');
query.on('result', function(row){
console.log('query1', row);
var query2 = conn.query('select id from books where id = ? ', [row.id]);
query2.on('result', function(row2){
console.log('query2', row2);
var query3 = conn.query('insert into test (id) values (?)', [row2.id]);
query3.on('result', function(row3){
console.log(row3);
});
});
});
query.on('end', function(){
conn.end();
});
The above fails, because there are still rows to process in query3 after the initial query is ended.
Any thoughts? The actual code is even more complicated, because I have to process xml from the subsequent queries and fire off even more inserts as I loop through the batch.
Thanks!
I would suggest this solution with async module:
var async = require("async");
// connection instance
var conn;
// here goes task serving logic
// if any async function should be finished before drain callback, push them into q
var solvers = {
query: function(q, task, row){
console.log('query1', row);
q.push({
solver: "query2",
req: "select id from books where id = ?",
reqArgs: [row.id]
});
},
query2: function(q, task, row){
console.log('query2', row);
q.push({
solver: "query3",
req: "insert into test (id) values (?)",
reqArgs: [row.id]
});
},
query3: function(q, task, row){
console.log(row);
}
}
// here is a queue of tasks
var q = async.queue(function(task, cb){
var query = conn.query(task.req, task.reqArgs);
query.on("end", cb);
query.on("result",function(row){
solvers[task.solver](q, task, row);
});
}, 2); // limit of parallel queries
// when every request has reached "end"
q.drain = function(){
conn.end();
// continue from here
};
// initial task
q.push({
solver: "query",
req: "select id from table1 limit 10",
reqArgs: []
});
But still, I'm not sure that making requests ID by ID is a good solution.
Maybe, I'm just not aware of a full problem.
#glukki, thanks for the great answer and reference to async. I went with a permutation of your code and two async requests which do a 'chomp and chew' using a single connection and pool of connections to process over 100K row select into 1.2M row inserts. Worked amazingly well and took less than 10 minutes. Here's the full implementation minus the module and connection setup. I hope this helps someone else too. Thanks again!
function populateMesh(row, callback){
xmlParser.parseString('<root>'+row.mesh_heading_list+'</root>', function(err, result){
var q2 = async.queue(function (task, cb) {
pool.getConnection(function(err, cnx){
cnx.query('INSERT INTO abstract_mesh (mesh_id, abstract_id, major_topic) SELECT mesh_descriptor.id, ?, ? FROM mesh_descriptor WHERE mesh_descriptor.name = ?', [task.id, task.majorTopic, task.descriptorName], function(err, result){
if (err) {throw err;}
cnx.release();
cb();
});
});
}, 50);
q2.drain = function() {
//console.log('all mesh processed');
callback();
}
if(!(result.root instanceof Object)){
//console.log('its not obj!', row.id);
q2.push({id: row.id, majorTopic: 'N', descriptorName: 'Null'}, function (err) {});
}
for(var i in result.root.MeshHeading){
// console.log('in loop',result.root.MeshHeading[i].DescriptorName);
if(typeof result.root.MeshHeading[i].DescriptorName === 'undefined'){
q2.push({id: row.id, majorTopic: 'N', descriptorName: 'Emergency'}, function(err){});
}
for(var j in result.root.MeshHeading[i].DescriptorName){
var descriptorName = result.root.MeshHeading[i].DescriptorName[j]._;
var majorTopic = result.root.MeshHeading[i].DescriptorName[j].$.MajorTopicYN;
q2.push({id: row.id, majorTopic: majorTopic, descriptorName: descriptorName}, function (err) {});
}
}
});
}
// here goes task serving logic
// if any async function should be finished before drain callback, push them into q
var q = async.queue(function (row, callback) {
console.log('got id: ' + row.id);
populateMesh(row, function(){
callback();
});
}, 10);
q.drain = function() {
console.log('all items have been processed');
conn.end(function(err){
console.log('connection ended');
});
pool.end(function(err){
console.log('pool closed');
});
};
var truncate = conn.query('truncate abstract_mesh');
var select = conn.query('SELECT id, mesh_heading_list FROM pubtbl');
select.on('result', function(result){
// console.log(result);
q.push(result, function (err) {
//console.log('finished processing row');
});
});
In my opinion the best solution is to make the code synchronously in a very easy way.
You could use the "synchonize" package.
Just
npm install synchronize
Then var sync = require(synchronize);
Put logic which should be synchronous into a fiber by using
sync.fiber(function() {
//put your logic here
}
An example for two mysql queries:
var express = require('express');
var bodyParser = require('body-parser');
var mysql = require('mysql');
var sync = require('synchronize');
var db = mysql.createConnection({
host : 'localhost',
user : 'user',
password : 'password',
database : 'database'
});
db.connect(function(err) {
if (err) {
console.error('error connecting: ' + err.stack);
return;
}
});
function saveSomething() {
var post = {id: newId};
//no callback here; the result is in "query"
var query = sync.await(db.query('INSERT INTO mainTable SET ?', post, sync.defer()));
var newId = query.insertId;
post = {foreignKey: newId};
//this query can be async, because it doesn't matter in this case
db.query('INSERT INTO subTable SET ?', post, function(err, result) {
if (err) throw err;
});
}
When "saveSomething()" is called, it inserts a row in a main table and receives the last inserted id. After that the code below will be executed. No need for nesting promises or stuff like that.
This is what i did,
db.query(
"select name from USER where name = ?",
["test"],
(err, result) => {
if (err) {
console.log("Error : ", err);
} else if (result.length <= 0) {
res.json("Not Found");
} else {
console.log("name found, executing update query!");
updateAgeIfUserFound("test"); //Calling funtion with 2nd query
}
}
);
//Update age only if name is present
function updateAgeIfUserFound(name, age) {
if (name) {
db.query(
"update USER set age = ? where name = ?,
[age, name],
(err, result) => {
if (err) throw err;
console.log("Name Updated");
res.json("Name Updated");
}
);
}
}