I am new to nodejs and I am trying to use multiple queries to display result in a single page. I have used async parallel to get the result but I am unable to handle the requests.
This is my callback function
gettournamentDetail: function (res, req) {
var connection = mysqlConnectionProvider.getSqlConnection();
//var test=[];
var collection = { collection:[] };
var sqlStatement = "SELECT * FROM tournaments WHERE tournamentId =" + res;
var tgroup1 = { collection:[]};
var tournamentdetails = [];
var sqlStatement2 = "select * from " +
"(SELECT DISTINCT teamId, teamName,ttournament FROM teams, tournamentTeam WHERE ttournament=" + res + ") as main LEFT JOIN (SELECT DISTINCT groupName,groupTournament FROM groupTeam,tournamentGroup WHERE groupt=tgId) as sub ON sub.groupTournament=main.ttournament";
// var return_data = {};
async.parallel([
function () {
if (connection) {
connection.query(sqlStatement, function (err, rows, fields) {
rows.forEach(function (row) {
tournamentdetails=row;
});
req(tournamentdetails);
});
}
},
function () {
var tgroup=[];
if (connection) {
connection.query(sqlStatement2, function (err, rows, fields) {
for (var i in rows) {
tgroup.push("group",{
teamName: rows[i].teamName,
teamId: rows[i].teamId,
groupName: rows[i].groupName
});
}
req(tgroup);
});
}
}
This is my routing page
exports.tournamentDetail = function( req, res) {
var tournamentdetail = require('../database/getTournament.js');
tournamentdetail.tournament.gettournamentDetail(req.params.id, function (collection) {
console.log(collection);
//res.render('tournamentdetail', {title: 'Tournament Detail ', tdetail: collection});
});
};
Any Idea how Can I handle the two requests to display the data.
This is how I am getting the data
RowDataPacket {
tournamentId: 1,
tournamentUser: 1,
tournamentDate: 1472299200,
tournamentLocation: 'Reading',
tournamentDesc: 'An alternative to renaming app.js is to create an elastic beanstalk configuration file. Add a .config file into the .ebextensions folder, for example, .ebextensions/34.config. Change the NodeCommand setting in the namespace aws:elasticbeanstalk:container:nodejs to whatever command you want to run to start the server. For example, this is a minimal .config file to run npm start instead of app.js:\n\n',
tournamentInfo: 'Test',
tournamentCreated: 1471171131,
tournamentName: 'Ping Pong' }
[ 'group',
{ teamName: 'TeamName', teamId: 1, groupName: 'A' },
'group',
{ teamName: 'Team2', teamId: 2, groupName: 'A' } ]
I little simplify you code, but I don't understand exactly what you needs.
// connect on application start
var connection = mysqlConnectionProvider.getSqlConnection();
if (!connection)
throw new Error('Smth wrong');
...
let getTournamentDetail = function (tournament_id, callback) {
// I think both queries are bad, because they used Cartesian product and distinct
// Use placeholder to escaping params; it's more safety
var sql = 'select * from tournaments where tournamentId = ??';
var sql2 = 'select distinct teamId, teamName, ttournament '+
'from teams, tournamentTeam WHERE ttournament = ??) as main ' +
'left join (select distinct groupName, groupTournament ' +
'from groupTeam, tournamentGroup where groupt = tgId) as sub ' +
'on sub.groupTournament = main.ttournament';
async.parallel([
function (callback) { connection.query(sql, [tournament_id], callback) },
function (callback) { connection.query(sql2, [tournament_id], callback) }
],
function (err, results) {
if (err)
return callback(err);
callback(null, {tournamentdetails: results[0], tgrup: results[1]})
}
);
}
...
var youRouteFunc = function (req, res) {
getTournamentDetail(req.params.id, function (err, data) {
if (err)
return res.send(err.message);
res.render('tournamentdetail', {title: 'Tournament Detail ', tdetail: data});
});
}
Related
I have a lambda function that connects to mysql and runs a set of queries, but I actually have a sequence of mysql queries that need to run one after another. I.e., the value of one query is used in the next query, etc.
Currently, I have a bunch of callbacks to achieve this, but this is leading to "callback hell". How would I rewrite this to use async / await?
My code is actually split into 2 files. The first file does an initial query, and then the value is passed into a function of the second file. Please note that the mysql node_module is included but not shown here. The AWS API gateway calls index.js
// index.js
var mysql = require('mysql'); // from node_modules
var config = require('./config.json');
var dashboard = require('./dashboard.js');
var pool = mysql.createPool({
host : config.dbhost,
user : config.dbuser,
password : config.dbpassword,
database : config.dbname
});
exports.handler = (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
pool.getConnection(function(err, connection) {
// check for mysql connection error first
if ( err ) {
throw err;
}
let qry = "select id from some_table where some_field = ?";
let someval = event.queryStringParameters.someval;
connection.query(qry, [someval], function(error, result) {
if ( error ) {
throw err;
}
else {
dashboard.processRequest(connection, callback, event, res[0].id);
}
});
});
}
// dashboard.js
module.exports = {
jsonResponse: function(results) {
return {
"statusCode": 200,
"body": JSON.stringify({ results }),
"isBase64Encoded": false,
"headers": {
"Access-Control-Allow-Origin": "*"
}
};
},
processRequest: function(connection, callback, event, val) {
let qry = "update first_table set some_field = ?";
connection.query(qry, [val], function(error, results) {
// return to client if error
if (error) {
callback(null, this.jsonResponse(error));
}
else {
// assume that this table must be update AFTER the previous statement
qry = "select id from second_table where some_field = ?";
connection.query(qry, [val], function(error1, results1) {
// return to client if error
if ( error1 ) {
callback(null, this.jsonResponse(error1));
}
qry = "update third_table set some_field = ? where id = ?";
connection.query(qry, [results1[0].id], function(error2, results2) {
// release connection when all queries are completed
connection.release();
if ( error2 ) {
callback(null, this.jsonResponse(error2));
}
else {
callback(null, this.jsonResponse(results2));
}
});
});
}
});
}
};
It was suggested to me that something like the below code might work. Unfortunately, it does not. I was curious to know why using try...catch blocks in the way shown below is not working, and is it the same thing as what you've shown, but just written differently?
// index.js
var mysql = require('mysql'); // from node_modules
var config = require('./config.json');
var dashboard = require('./dashboard.js');
var pool = mysql.createPool({
host : config.dbhost,
user : config.dbuser,
password : config.dbpassword,
database : config.dbname
});
exports.handler = (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
pool.getConnection(function(err, connection) {
// check for mysql connection error first
if ( err ) {
throw err;
}
let qry = "select id from users where username = ? limit 1;";
let username = event.queryStringParameters.username;
try {
let res = await connection.query(qry, [event.queryStringParameters.username]);
dashboard.processRequest(connection, callback, event, res[0].id);
} catch (err) {
console.log(err);
}
});
}
// dashboard.js
module.exports = {
jsonResponse: function (results) {
return {
"statusCode": 200,
"body": JSON.stringify({results}),
"isBase64Encoded": false,
"headers": {
"Access-Control-Allow-Origin": "*"
}
};
},
processRequest: async function (connection, callback, event, val) {
let qry = "update first_table set some_field = ?";
try {
let results = await connection.query(qry, [val]);
qry = "select id from second_table where some_field = ?";
try {
let results1 = await connection.query(qry, [val]);
qry = "update third_table set some_field = ? where id = ?";
try {
let results2 = await connection.query(qry, [results1[0].id]);
connection.release();
callback(null, this.jsonResponse(results2));
} catch (error2) {
callback(null, this.jsonResponse(error2));
}
} catch (error1) {
callback(null, this.jsonResponse(error1));
}
} catch (error) {
callback(null, this.jsonResponse(error));
}
}
};
We need use promises.
Typically I follow this approach:
Create one async method mainProcess and have bunch of methods step by step called with in that method. one after the other with await or all at once.
Each async method getConnection and runQuery in this case, called within mainProcess must a Promise.
If any errors from these methods i.e promise rejects from individual methods, goes in catch block of mainProcess().
If no errors, all methods within mainProcess gets executed and goes to then block of mainProcess()
Your code can be refactored like this (just wrote in an editor untested)
var pool = mysql.createPool({
host: config.dbhost,
user: config.dbuser,
password: config.dbpassword,
database: config.dbname,
});
exports.handler = (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
/**
* Main Lambda Process
*/
const mainProcess = async () => {
// Get Connection
let connection = await getConnection();
// Run Step 1
let qry1 = "select id from some_table1 where some_field = ?";
const response1 = await runQuery(connection, qry1, { someFiledValue: 1222})
// Run Step 2
let qry2 = "select id from some_table2 where some_field = ?";
const resonse2 = await runQuery(connection, qry2, { someFiledValue: 1222})
return 'All Good';
});
}
mainProcess()
.then(result => {
// All lambda success messages are returned from here
callback(null, result);
})
.catch(error => {
// All lambda errors thrown from here
callback(error);
});
};
function getConnection(qry, parms) {
return new Promise((resolve, reject) => {
pool.getConnection(function (error, connection) {
if (error) {
// return to client if error
reject(error);
} else {
// Return response
resolve(connection);
}
});
});
}
/**
* Runs a query, either resolves or rejects
*/
function runQuery(connection, qry, parms) {
return new Promise((resolve, reject) => {
connection.query(qry, [val], function (error, results) {
if (error) {
// return to client if error
reject(error);
} else {
// Return response
resolve(result);
}
});
});
}
When you're dealing with a lambda function which performs an async task you have two solutions:
you can use non async handlers, in which case you need to invoke "callback" on promises as you did in your example
you can use async handlers, which does not requires the "callback" input and that allows you to write async/await code, like the following example:
const mysql = require('mysql2/promise');
exports.handler = async(event, context) => {
//get path variable
const { pathVar } = event.pathParameters;
// get connection
const connection = await mysql.createConnection({
host : process.env.RDS_HOSTNAME,
user : process.env.RDS_USERNAME,
password : process.env.RDS_PASSWORD,
database : process.env.RDS_DB_NAME
});
// get text query
const textQuery = `SELECT field FROM entity WHERE attribute = ${pathVar}`;
// get res
const results = await connection.execute(textQuery);
return {
"statusCode": 200,
"body": results,
"isBase64Encoded": false
}
}
You can have a look at the AWS docs: https://docs.aws.amazon.com/lambda/latest/dg/nodejs-handler.html
I am trying to iterate through a MySQL query result and make subsequent queries in order to build out my data model. Each object requires multiple queries, therefore I am chaining promises.
The problem occurs when I nest a second set of promises.
So first I am getting a list of the objects that need to be retrieved using g.getSnapshotIds. Then I iterate through those and use the snapshotId to retrieve a full snapshot.
var gData = {};
g.getSnapshotIds(data.gId, data.userId)
.then(function(value) {
gData = value;
for ( var snapshot in value ) {
var snapshotId = value[snapshot].snapshotId;
var snapshot = {};
g.getSnapshotFull(snapshotId)
.then(function(value) {
console.log(value);
return g.getTs(snapshotId);
})
.then(function(value) {
for ( var te in value ) {
var name = value[te].t;
snapshot[name] = value[te].value;
}
console.log(snapshot);
})
.catch(function(err) {
console.log('Error:', err);
});
}
g.close();
})
.catch(function(err) {
console.log('Error:', err);
});
I am able to call g.getSnapshotFull on each ID, but when I try to move on to the next query (g.getTs(snapshotId)) it gives me the error:
Error: Cannot enqueue Query after invoking quit.
I have no idea why the MySQL connection is closing before all queries are done. Shouldn't everything inside the for loop execute sequentially before moving on?
If I comment out g.close(), I don't get the error, but the process doesn't end.
These are the relevant query methods:
class gDB {
close() {
return new Promise(function(resolve, reject) {
db.end(function(error) {
if ( error ){
reject(new Error(error));
}
// For some reason it is necessary to reestablish this
db = mysql.createConnection({
host: process.env.DBHOST,
user: process.env.DBUSER,
password: process.env.DBPASS,
database: process.env.DBNAME,
ssl: {
ca: fs.readFileSync(__dirname + '/' + process.env.DBCA)
}
});
resolve(true);
});
});
}
getSnapshotIds(gId, uId) {
return new Promise(function(resolve, reject) {
var sql = 'SELECT id AS snapshotId FROM snapshots WHERE gId=' + db.escape(gId) + ' AND uId=' + db.escape(uId) + ' ORDER BY timestamp DESC';
db.query(sql, function (error, results, fields) {
if (error) {
db.destroy();
reject(new Error(error));
} else {
resolve(results);
}
});
});
}
getSnapshotFull(snapshotId) {
return new Promise(function(resolve, reject) {
var sql = 'SELECT s.id AS snapshotId, s.timestamp, s.gId, s.uId, s.clientId FROM snapshots s INNER JOIN controls c ON s.id = c.snapshotId INNER JOIN weathers w ON s.id = w.snapshotId WHERE s.id=' + db.escape(snapshotId);
db.query(sql, function (error, results, fields) {
if (error) {
db.destroy();
reject(new Error(error));
} else {
resolve(results[0]);
}
});
});
}
getTs(snapshotId) {
return new Promise(function(resolve, reject) {
var sql = 'SELECT t.t, st.value FROM snapshots s LEFT JOIN snapshot_t st ON s.id = st.snapshotId INNER JOIN ts t ON st.tId = t.id WHERE s.id=' + db.escape(snapshotId);
db.query(sql, function (error, results, fields) {
if (error) {
db.destroy();
reject(new Error(error));
} else {
resolve(results);
}
});
});
}
The problem you are having is for loops are synchronous while promises are asynchronous. What is going on is you are creating a bunch of promises that are waiting for something to happen (the promise to receive data), then the for loop ends (before any of the promises finish) and you then call close. What you'll want to do is something similar to the below.
var gData = {};
g.getSnapshotIds(data.gId, data.userId)
.then(function (value) {
gData = value;
var promises = [];
for (var snapshot in value) {
var snapshotId = value[snapshot].snapshotId;
var snapshot = {};
var promise = g.getSnapshotFull(snapshotId)
.then(function (value) {
console.log(value);
return g.getTs(snapshotId);
})
.then(function (value) {
for (var te in value) {
var name = value[te].t;
snapshot[name] = value[te].value;
}
console.log(snapshot);
});
promises.push(promise);
}
return Promise.all(promises);
})
.then(function (values) {
g.close();
console.log(values);
})
.catch(function (err) {
console.log('Error:', err);
});
What solves this is saving the promise and then using Promise.all(promises) to wait for all the promises to finish. The last then block will have the results of all of the promises and that is where you can close your database connection.
Am trying to retrieve all the database records from a table called post using node js but the problem is that only one record is retrieved instead of all.
In php I can use while() loop to loop through the database record to get all data.
Currently, I do not know how to neatly loop through the database in nodejs to get all the records from database. Some Stackoverflow scholars suggest using await/async method but i do not know to to implement it on the code below to make it work. can someone help me fix the issue.
var connection = require('./config');
module.exports.getpost = function (req, res) {
connection.query('SELECT * FROM posts', function (error, results, fields) {
if (error) {
console.log('error');
res.json({
status : false,
message : 'there are some error with the query'
});
} else {
var postid = results[0].id;
var title = results[0].title;
var content = results[0].content;
var type = -1;
console.log(title);
// Checking user status
connection.query('SELECT count(*) as cntStatus,type FROM like_table WHERE userid= ? and postid=?', [userid,postid], function (error, results, fields) {
if (error) {
console.log('error');
res.json({
status : false,
message : 'there are some error with the query'
});
} else {
var total_count = results[0].cntStatus;
if(total_count > 0){
type = results[0].type;
}
var total_count = results[0].cntStatus;
var result = {
"id" : postid,
"title" : title,
"content" : content,
"type" : type,
"likes" : total_count
};
console.log('query okay');
res.json({
//data:results,
data : result
});
}
});
}
});
}
I'm assuming you're using mysql npm. In that case I'm not sure what is the problem in your case. Results param is an array of rows returned by your select statement. So you can use loop to iterate trough all the rows.
You don't actually need to use async/await (which doesn't have any advantage in terms of functionality but looks cleaner). But if you want to get rid of callbacks you need to wrap connection query into a promise or use mysql2 npm which has promise interface. Here is how you can iterate trough all the rows from your select using async/await instead of callback:
var connection = require('./config');
module.exports.getpost = async function (req, res) {
try {
const queryResult = await query('SELECT * FROM posts');
queryResult.forEach(row => {
console.log(row.title);
})
} catch (err) {
console.log('error');
res.json({
status: false,
message: 'there are some error with the query'
});
}
}
Please note that you need to use nodejs 8 to run the code with async/await.
Also you don't need to do another query inside of your posts query, you can merge those two using SQL join
async waterfall - Runs an array of functions in series, each passing their results to the next in the array. However, if any of the functions pass an error to the callback, the next function is not executed and the main callback is immediately called with the error.
var connection = require('./config');
var async = require('async');
module.exports.getpost = function (req, res) {
var arrayOfFuncs = [];
var func_1 = function(callback) {
connection.query('SELECT * FROM posts', function (error, results, fields) {
if (error) {
console.log('error');
callback(error, null);
} else {
var toPass = {};
toPass.postid = results[0].id;
toPass.title = results[0].title;
toPass.content = results[0].content;
toPass.type = -1;
callback(null, toPass);
}
})
}
arrayOfFuncs.push(func_1);
var func_2 = function(prevData, callback) {
connection.query('SELECT count(*) as cntStatus,type FROM like_table WHERE userid= ? and postid=?', [userid,prevData.postid], function (error, results, fields) {
if (error) {
console.log('error');
callback(error, null);
} else {
var total_count = results[0].cntStatus;
if(total_count > 0){
type = results[0].type;
}
var total_count = results[0].cntStatus;
var result = {
"id" : postid,
"title" : title,
"content" : content,
"type" : type,
"likes" : total_count
};
console.log('query okay');
callback(null, result);
}
});
}
arrayOfFuncs.push(func_2);
async.waterfall(arrayOfFuncs, function(errString, finalResult) {
if(errString) {
return res.send(errString);
} else {
return res.send(finalResult);
}
});
}
Im using expressjs and mysql package.
I have a user table, each user could have many skills, projects, certifications stored in different tables. How can I get all of these data into a single user object.
getUserDataById(req.params.id, function (user) {
if (user) {
res.send(user);
}
}
var getUserDataById = function (id, callback) {
connection.query(userSql, id, function (err, rows) {
if (rows[0]) {
user = new User(rows[0]); //parse row data to user obj
// The main problem
//get skills and projects.... and asign to user obj
parseArrayData(skillSql, user.id, Skill, function (skills) {
user.skills = skills;
}
);
parseArrayData(projectSql, user.id, Project, function (project) {
user.projects = projects;
}
);
// is here
callback(user);
}
});
}
var parseArrayData = function (query, id, Obj, callback) {
connection.pool.query(query, id, function (err, rows) {
if (err) {
throw err;
} else {
console.log('rows',rows);
var array = [];
for (var i = 0; i < rows.length; i++) {
array[i] = new Obj(rows[i]); // map obj's attributes with fields
};
callback(rows);
}
});
};
You might have these different tables with names (assume): users, skills, projects, certifications;
function to get result from db for query;
function executeQuery(tablename, userId, callback) {
var sql = 'select * from ' + tablename + ' where id = ' + userId;
connection.query(sql, function (err, rows) {
if (err) { callback(err, null); }
else { callback(null, rows); }
})
}
Function that will fetch all results for user id and return json object;
var getUserDataById = function (userId, callback) {
executeQuery('users', userId, function(users) {
executeQuery('projects', userId, function(projects) {
executeQuery('skills', userId, function(skills) {
executeQuery('certifications', userId, function(certifications) {
var result = { // format you result.
id: userId,
users: users,
projects: projects,
skills: skills,
certifications: certifications
};
callback(result);
});
});
});
});
callback(null);
});
Usage:
getUserDataById(req.params.id, function (user) {
if (user) {
res.send(user);
}
}
I'm new in making API. I use Node.js and MySQL.
The fact is I have two GET function to get all users and one to get user by ID.
Both function are working when they are alone implemented. If both of them are implemented the function to get all user try to enter in the function to get user by ID so the API crash.
So here is my model users.js
var connection = require("../connection");
function Users()
{
//GET ALL USERS
this.get = function(res)
{
console.log('Request without id');
connection.acquire(function(err, con)
{
con.query('SELECT * FROM users', function(err, result)
{
con.release();
if (err)
res.send({status: 1, message: 'Failed to get users'})
else
res.send(result);
});
});
}
//GET USER BY ID
this.get = function(id, res)
{
console.log('Request with ID');
connection.acquire(function(err, con)
{
if (id != null)
{
con.query('SELECT * FROM users WHERE id = ?', id, function(err, result)
{
con.release();
if (err)
res.send({status: 1, message: 'Failed to find user: ' + id});
else if (result == "")
res.send({status: 1, message: 'Failed to find user: ' + id});
else
res.send(result);
});
}
});
}
And here is the routes.js
var users = require('./models/users');
module.exports = {
configure: function(app) {
app.get('/users/', function(req, res) {
users.get(res);
});
app.get('/users/:id/', function(req, res) {
users.get(req.params.id, res);
});
Do you have any idea why ?
Thanks for help :)
You can't have two functions with the same name in the same scope.
You have to rename your functions
/**
* Get all users
*/
this.get = function(res) {...}
/**
* Get user by id
*/
this.getById = function(id, res) {...}
Or you can have one function and check if an id is provided
this.get = function(id, res) {
if ( Number.isInteger(id) ) {
// return the user
} else {
res = id;
// return all users
}
}