Node mysql and promises - mysql

I have a db module in my node worker. All I need is make it straight, without callbacks. query method still returns undefined. I suppose I did something very wrong in this piece of code but I have never seen the full implementation of promises in one place.
db: {
init: function (parent) {
var self = this;
self.parent = parent;
self.pool = mysql.createPool({
host: self.parent.cfg['mysql']['host'].replace(/"/g, ''),
user: self.parent.cfg['mysql']['username'].replace(/"/g, ''),
port: self.parent.cfg['mysql']['port'].replace(/"/g, ''),
password: self.parent.cfg['mysql']['password'].replace(/"/g, ''),
database: self.parent.cfg['mysql']['db'].replace(/"/g, '')
});
self._query = Q.nbind(self.__query, self);
},
query: function (query, params) {
var self = this;
return self._query(query, params).then(function (rows) {
return rows;
}).catch(function(err){
return err;
}).done();
},
__query: function (query, params) {
var self = this, deferred;
deferred = Q.defer();
query = params ? mysql.format(query, params) : query;
//console.log(query);
self.pool.getConnection(function (err, connection) {
if (err) {
deferred.reject(err);
}
connection.query(query, function (err, rows) {
connection.release();
if (err) {
deferred.reject(err);
}
deferred.resolve(rows);
});
});
return deferred.promise;
}
}
dump of deferred.promise
Most likely that Q.nbind is not necessary here, anyway I dumped everything and deferred.promise is an object {state: 'pending'}. So my opinion the devil is in __query function.
added
query: function (query, params) {
var self = this;
return self._query(query, params).then(function (rows) {
console.log(rows);
return rows;
}).catch(function(err){
return err;
}).done();
}
it puts rows to console, but doesn't return it. One guy claimed that such structure works:
function getKey(key) {
return r.get(dbkey(key)).then(function(x) {
return x && parseInt(x, 10);
});
}
added again
Yes, removing done() makes sense. But it still returns {state: 'pending'}. May be the problem is in the way I am calling query?
test: function () {
var self = this;
var s = self.db.query(self.testQuery);
console.log(s); // { state: 'pending' }
},

I believe you problem is how you are using Q.nbind. It is intended for functions that make a node style callback function(err, result ...
You are dealing with this in your __query function with the deferred. So I think you should just use __query directly and you will be in business. I'll post a code example when I'm back on my computer.

Ok, solution
db: {
init: function (parent) {
var self = this;
self.parent = parent;
self.pool = mysql.createPool({
host: self.parent.cfg['mysql']['host'].replace(/"/g, ''),
user: self.parent.cfg['mysql']['username'].replace(/"/g, ''),
port: self.parent.cfg['mysql']['port'].replace(/"/g, ''),
password: self.parent.cfg['mysql']['password'].replace(/"/g, ''),
database: self.parent.cfg['mysql']['db'].replace(/"/g, '')
});
},
query: function (query, params) {
var self = this;
return self._query(query, params).then(function (rows) {
return rows;
}).catch(function (err) {
return err;
});
},
_query: function (query, params) {
var self = this, deferred;
deferred = Q.defer();
query = params ? mysql.format(query, params) : query;
//console.log(query);
self.pool.getConnection(function (err, connection) {
if (err) {
deferred.reject(err);
}
connection.query(query, function (err, rows) {
connection.release();
if (err) {
deferred.reject(err);
}
deferred.resolve(rows);
});
});
return deferred.promise;
}
}
And how to call it:
test: function () {
var self = this;
Q.async(function*() {
var s = yield self.db.query(self.testsql);
console.log(s); // it is real rows
})().done();
},
That is really powerful thing, you can write methods which returns promises and then use them all in one place you want:
Q.async(function*() {
var device = yield self.modules.ami.device.get('123');
var ext = yield self.modules.ami.extension.get('123');
})().done();
While the ami.device.get and ami.extension.get method returns db.query promises:
get: function(id){
var self = this;
id = parseInt(id);
return self.top.db.query(self.select + "WHERE (`ami_device`.`id` = ?)", [id]);
}
I have to say thanks a lot to Gordon Bockus for help. This guy really saved me a day!

Related

convert a NodeJS lambda function (AWS) to use "async" (promises, etc.) instead of callbacks

I have a lambda function that connects to mysql and runs a set of queries, but I actually have a sequence of mysql queries that need to run one after another. I.e., the value of one query is used in the next query, etc.
Currently, I have a bunch of callbacks to achieve this, but this is leading to "callback hell". How would I rewrite this to use async / await?
My code is actually split into 2 files. The first file does an initial query, and then the value is passed into a function of the second file. Please note that the mysql node_module is included but not shown here. The AWS API gateway calls index.js
// index.js
var mysql = require('mysql'); // from node_modules
var config = require('./config.json');
var dashboard = require('./dashboard.js');
var pool = mysql.createPool({
host : config.dbhost,
user : config.dbuser,
password : config.dbpassword,
database : config.dbname
});
exports.handler = (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
pool.getConnection(function(err, connection) {
// check for mysql connection error first
if ( err ) {
throw err;
}
let qry = "select id from some_table where some_field = ?";
let someval = event.queryStringParameters.someval;
connection.query(qry, [someval], function(error, result) {
if ( error ) {
throw err;
}
else {
dashboard.processRequest(connection, callback, event, res[0].id);
}
});
});
}
// dashboard.js
module.exports = {
jsonResponse: function(results) {
return {
"statusCode": 200,
"body": JSON.stringify({ results }),
"isBase64Encoded": false,
"headers": {
"Access-Control-Allow-Origin": "*"
}
};
},
processRequest: function(connection, callback, event, val) {
let qry = "update first_table set some_field = ?";
connection.query(qry, [val], function(error, results) {
// return to client if error
if (error) {
callback(null, this.jsonResponse(error));
}
else {
// assume that this table must be update AFTER the previous statement
qry = "select id from second_table where some_field = ?";
connection.query(qry, [val], function(error1, results1) {
// return to client if error
if ( error1 ) {
callback(null, this.jsonResponse(error1));
}
qry = "update third_table set some_field = ? where id = ?";
connection.query(qry, [results1[0].id], function(error2, results2) {
// release connection when all queries are completed
connection.release();
if ( error2 ) {
callback(null, this.jsonResponse(error2));
}
else {
callback(null, this.jsonResponse(results2));
}
});
});
}
});
}
};
It was suggested to me that something like the below code might work. Unfortunately, it does not. I was curious to know why using try...catch blocks in the way shown below is not working, and is it the same thing as what you've shown, but just written differently?
// index.js
var mysql = require('mysql'); // from node_modules
var config = require('./config.json');
var dashboard = require('./dashboard.js');
var pool = mysql.createPool({
host : config.dbhost,
user : config.dbuser,
password : config.dbpassword,
database : config.dbname
});
exports.handler = (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
pool.getConnection(function(err, connection) {
// check for mysql connection error first
if ( err ) {
throw err;
}
let qry = "select id from users where username = ? limit 1;";
let username = event.queryStringParameters.username;
try {
let res = await connection.query(qry, [event.queryStringParameters.username]);
dashboard.processRequest(connection, callback, event, res[0].id);
} catch (err) {
console.log(err);
}
});
}
// dashboard.js
module.exports = {
jsonResponse: function (results) {
return {
"statusCode": 200,
"body": JSON.stringify({results}),
"isBase64Encoded": false,
"headers": {
"Access-Control-Allow-Origin": "*"
}
};
},
processRequest: async function (connection, callback, event, val) {
let qry = "update first_table set some_field = ?";
try {
let results = await connection.query(qry, [val]);
qry = "select id from second_table where some_field = ?";
try {
let results1 = await connection.query(qry, [val]);
qry = "update third_table set some_field = ? where id = ?";
try {
let results2 = await connection.query(qry, [results1[0].id]);
connection.release();
callback(null, this.jsonResponse(results2));
} catch (error2) {
callback(null, this.jsonResponse(error2));
}
} catch (error1) {
callback(null, this.jsonResponse(error1));
}
} catch (error) {
callback(null, this.jsonResponse(error));
}
}
};
We need use promises.
Typically I follow this approach:
Create one async method mainProcess and have bunch of methods step by step called with in that method. one after the other with await or all at once.
Each async method getConnection and runQuery in this case, called within mainProcess must a Promise.
If any errors from these methods i.e promise rejects from individual methods, goes in catch block of mainProcess().
If no errors, all methods within mainProcess gets executed and goes to then block of mainProcess()
Your code can be refactored like this (just wrote in an editor untested)
var pool = mysql.createPool({
host: config.dbhost,
user: config.dbuser,
password: config.dbpassword,
database: config.dbname,
});
exports.handler = (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
/**
* Main Lambda Process
*/
const mainProcess = async () => {
// Get Connection
let connection = await getConnection();
// Run Step 1
let qry1 = "select id from some_table1 where some_field = ?";
const response1 = await runQuery(connection, qry1, { someFiledValue: 1222})
// Run Step 2
let qry2 = "select id from some_table2 where some_field = ?";
const resonse2 = await runQuery(connection, qry2, { someFiledValue: 1222})
return 'All Good';
});
}
mainProcess()
.then(result => {
// All lambda success messages are returned from here
callback(null, result);
})
.catch(error => {
// All lambda errors thrown from here
callback(error);
});
};
function getConnection(qry, parms) {
return new Promise((resolve, reject) => {
pool.getConnection(function (error, connection) {
if (error) {
// return to client if error
reject(error);
} else {
// Return response
resolve(connection);
}
});
});
}
/**
* Runs a query, either resolves or rejects
*/
function runQuery(connection, qry, parms) {
return new Promise((resolve, reject) => {
connection.query(qry, [val], function (error, results) {
if (error) {
// return to client if error
reject(error);
} else {
// Return response
resolve(result);
}
});
});
}
When you're dealing with a lambda function which performs an async task you have two solutions:
you can use non async handlers, in which case you need to invoke "callback" on promises as you did in your example
you can use async handlers, which does not requires the "callback" input and that allows you to write async/await code, like the following example:
const mysql = require('mysql2/promise');
exports.handler = async(event, context) => {
//get path variable
const { pathVar } = event.pathParameters;
// get connection
const connection = await mysql.createConnection({
host : process.env.RDS_HOSTNAME,
user : process.env.RDS_USERNAME,
password : process.env.RDS_PASSWORD,
database : process.env.RDS_DB_NAME
});
// get text query
const textQuery = `SELECT field FROM entity WHERE attribute = ${pathVar}`;
// get res
const results = await connection.execute(textQuery);
return {
"statusCode": 200,
"body": results,
"isBase64Encoded": false
}
}
You can have a look at the AWS docs: https://docs.aws.amazon.com/lambda/latest/dg/nodejs-handler.html

Lambda function MySQL result not working on NodeJs 8.10

I have a code in Node 6.10 and it is working...
But If I convert it to Node 8.10 it's not working
var mysql = require("mysql");
var connection = mysql.createConnection({
host: " localhost",
user: "root",
password: "",
database: "parser_db"
});
exports.handler = async event => {
connection.connect();
let response = {
statusCode: 400,
body: { Method: "Invalid", event }
};
var readTable = "SELECT * FROM documents where id = " + mysql.escape(1);
connection.query(readTable, function(err, results, fields) {
if (err) throw err;
else {
response = {
statusCode: 200,
body: { results }
//body: { results }
};
console.log(response);
return response;
}
});
};
Can some one please help me to detect the problem. It is also not working if I do the MySQL query in separate file and return the result set.
Note : If I print the result using console.log(response) instead returning it's
showing the correct result.
The problem is that you are returning response from within the connection.query() callback function. That makes response the return value for the callback function, not the return value for the outer Lambda function.
One way to restructure this code is as follows:
exports.handler = async (event) => {
connection.connect();
return new Promise((resolve, reject) => {
const readTable = `SELECT * FROM documents where id = ${mysql.escape(1)}`;
connection.query(readTable, (err, results, fields) => {
if (err) {
reject(err);
} else {
resolve({statusCode: 200, body: {results}});
}
});
});
};
In addition to #jarmod's answer, You can also use the util.promisify method to promisify connection.query so that you can use the await keyword, to make the code simpler
const util = require('util');
exports.handler = async (event) => {
connection.connect();
const readTable = `SELECT * FROM documents where id = ${mysql.escape(1)}`;
const connQueryPromisified = util.promisify(connection.query).bind(connection);
const result = await connQueryPromisified(readTable);
return {statusCode: 200, body: {results}};
};

Cannot enqueue Query after invoking quit when nesting promises

I am trying to iterate through a MySQL query result and make subsequent queries in order to build out my data model. Each object requires multiple queries, therefore I am chaining promises.
The problem occurs when I nest a second set of promises.
So first I am getting a list of the objects that need to be retrieved using g.getSnapshotIds. Then I iterate through those and use the snapshotId to retrieve a full snapshot.
var gData = {};
g.getSnapshotIds(data.gId, data.userId)
.then(function(value) {
gData = value;
for ( var snapshot in value ) {
var snapshotId = value[snapshot].snapshotId;
var snapshot = {};
g.getSnapshotFull(snapshotId)
.then(function(value) {
console.log(value);
return g.getTs(snapshotId);
})
.then(function(value) {
for ( var te in value ) {
var name = value[te].t;
snapshot[name] = value[te].value;
}
console.log(snapshot);
})
.catch(function(err) {
console.log('Error:', err);
});
}
g.close();
})
.catch(function(err) {
console.log('Error:', err);
});
I am able to call g.getSnapshotFull on each ID, but when I try to move on to the next query (g.getTs(snapshotId)) it gives me the error:
Error: Cannot enqueue Query after invoking quit.
I have no idea why the MySQL connection is closing before all queries are done. Shouldn't everything inside the for loop execute sequentially before moving on?
If I comment out g.close(), I don't get the error, but the process doesn't end.
These are the relevant query methods:
class gDB {
close() {
return new Promise(function(resolve, reject) {
db.end(function(error) {
if ( error ){
reject(new Error(error));
}
// For some reason it is necessary to reestablish this
db = mysql.createConnection({
host: process.env.DBHOST,
user: process.env.DBUSER,
password: process.env.DBPASS,
database: process.env.DBNAME,
ssl: {
ca: fs.readFileSync(__dirname + '/' + process.env.DBCA)
}
});
resolve(true);
});
});
}
getSnapshotIds(gId, uId) {
return new Promise(function(resolve, reject) {
var sql = 'SELECT id AS snapshotId FROM snapshots WHERE gId=' + db.escape(gId) + ' AND uId=' + db.escape(uId) + ' ORDER BY timestamp DESC';
db.query(sql, function (error, results, fields) {
if (error) {
db.destroy();
reject(new Error(error));
} else {
resolve(results);
}
});
});
}
getSnapshotFull(snapshotId) {
return new Promise(function(resolve, reject) {
var sql = 'SELECT s.id AS snapshotId, s.timestamp, s.gId, s.uId, s.clientId FROM snapshots s INNER JOIN controls c ON s.id = c.snapshotId INNER JOIN weathers w ON s.id = w.snapshotId WHERE s.id=' + db.escape(snapshotId);
db.query(sql, function (error, results, fields) {
if (error) {
db.destroy();
reject(new Error(error));
} else {
resolve(results[0]);
}
});
});
}
getTs(snapshotId) {
return new Promise(function(resolve, reject) {
var sql = 'SELECT t.t, st.value FROM snapshots s LEFT JOIN snapshot_t st ON s.id = st.snapshotId INNER JOIN ts t ON st.tId = t.id WHERE s.id=' + db.escape(snapshotId);
db.query(sql, function (error, results, fields) {
if (error) {
db.destroy();
reject(new Error(error));
} else {
resolve(results);
}
});
});
}
The problem you are having is for loops are synchronous while promises are asynchronous. What is going on is you are creating a bunch of promises that are waiting for something to happen (the promise to receive data), then the for loop ends (before any of the promises finish) and you then call close. What you'll want to do is something similar to the below.
var gData = {};
g.getSnapshotIds(data.gId, data.userId)
.then(function (value) {
gData = value;
var promises = [];
for (var snapshot in value) {
var snapshotId = value[snapshot].snapshotId;
var snapshot = {};
var promise = g.getSnapshotFull(snapshotId)
.then(function (value) {
console.log(value);
return g.getTs(snapshotId);
})
.then(function (value) {
for (var te in value) {
var name = value[te].t;
snapshot[name] = value[te].value;
}
console.log(snapshot);
});
promises.push(promise);
}
return Promise.all(promises);
})
.then(function (values) {
g.close();
console.log(values);
})
.catch(function (err) {
console.log('Error:', err);
});
What solves this is saving the promise and then using Promise.all(promises) to wait for all the promises to finish. The last then block will have the results of all of the promises and that is where you can close your database connection.

NodeJS/MySQL/Promises Troubles

I'm quite new to NodeJS and JS globally and I'm in trouble while setting and Object Property through a MySQL query.
I'm using Promise to avoid bad asynchronous effect but apparently I'm doing it wrong, the property of my Agent Obejct is never updated.
Here is the code :
class Agent {
constructor(agentId, agentName, agentCountry) {
this.agentId = agentId;
this.agentName = agentName;
this.agentCountry = agentCountry;
}
setAgentCountry () {
var promise = function(agentID) {
return new Promise(function(resolve, reject) {
var query = "SELECT c.CountryID, c.CountryName FROM AgentCountry ac, Country c WHERE ac.AgentID = '" + agentID + "' AND ac.CountryID = c.CountryID";
connection.query(query, function(err, results) {
if (!err) {
resolve(results);
} else {
console.log('Error while performing Query.');
}
});
});
}
promise(this.agentID).then(function(data) {
var string = JSON.stringify(data);
var json = JSON.parse(string);
//the agent property is never updated !!
this.agentCountry = json;
}.bind(this), function(err) {
console.log(err);
});
}
}
I call the method this way :
var agent = new Agent(1,"John Doe", "France");
console.log(agent.agentCountry); //Displays "France"
agent.setAgentCountry();
console.log(agent.agentCountry); //Did not display the table of countries it should
Could you help me with this ?
Thanks
The main problem is that console.log is being executed before the promise being resolved. Writing a console.log inside the "then" clause will show you the timing.
The promise will be resolved or rejected eventually but nobody is waiting for setAgentCountry.
There are several points of order here:
A promise must always be either (1) resolved or (2) rejected. Your error case logs it to the console without calling reject(), so it's stuck in promise limbo for forever when it errors.
Why do you name a variable, promise, the same as the library, Promise?
I think you will find it more modular to just wrap the mysql_conn.query() callback into a promise():
const mysql_conn = mysql.createConnection({
host: mysql_conf.host,
user: mysql_conf.user,
password: mysql_conf.password
});
mysql_conn.queryPromiser = function(sql, args) {
return new Promise(function(resolve, reject) {
mysql_conn.query(
sql,
args,
function(err, results, fields) {
if (err) {
reject(err);
} else {
resolve( {"results": results, "fields": fields} );
}
}
);
});
};
then you can use it like so:
class Agent {
constructor(agentId, agentName) {
this.agentId = agentId;
this.agentName = agentName;
this.agentCountry = null;
}
configureCountryPromiser() {
var sql = "SELECT country FROM agent_countries WHERE agent_id = ?";
var args = [ this.agentId ];
var that = this;
return mysql_conn.queryPromiser(sql, args)
.then(function(data) {
if (data.results.length) {
that.agentCountry = data.results[0].country;
} else {
// handle case where agent_id is not found in agent_countries
}
});
}
};
agent_instance = new Agent(1, "Benoit Duprat");
agent_instance.configureCountryPromiser()
.then(function() {
console.log("agent country configured to ", agent_instance.agentCountry);
}).catch(console.error);
Please note that I have not tested the class code, but the general idea should be enough to answer your question.

How to get the results from nodejs using mysql package?

first, i connect the db and select DB:
var defaultOptions = {
user: "root",
pwd:'admin',
db:"britcham_dev_local",
server:"local", // Maybe we don't need this variable.
};
var client = new Client();
client.user = defaultOptions.user;
client.password = defaultOptions.pwd;
client.connect(function (error, results) {
//
});
client.query('USE ' + defaultOptions.db, function (error, results) {
//
});
Second, I query with client object:
var self = this;
var this.users;
client.query("SELECT * FROM users", function (error, results, fields) {
if (error) {
//
}
if (results.length > 0) {
self.users = results;
}
});
console.log(this.users);
it's nothing output ??? Why ??
Since node.js is non-blocking and asynchronous, then in this code:
client.query("SELECT * FROM users", function (error, results, fields) {
if (error) {
//
}
if (results.length > 0) {
self.users = results;
}
});
console.log(this.users);
data from DB are not probably loaded yet into users variable when you are trying to log it into console. You can check it out if you do your console.log operation within the query, for example:
client.query("SELECT * FROM users", function (error, results, fields) {
if (error) {
//
}
if (results.length > 0) {
console.log(results);
}
});
To pass the result into a variable when the operation is finished you can wrap your client DB call into a function with callback parameter and set your variable when the callback is invoked, for example:
function query(sql, callback) {
client.query(sql, function (error, results, fields) {
if (error) {
//
}
if (results.length > 0) {
callback(results);
}
});
}
query("SELECT * FROM users", function(results) {
self.users = results;
console.log(self.users);
});
Above code is just a concept.
How is the suggested answer different from this?
var self = this;
var this.users;
client.query("SELECT * FROM users", function (error, results, fields) {
if (error) {
//
}
if (results.length > 0) {
self.users = results;
console.log(this.users);
}
});
I might be wrong this is not different from the suggested answer in that it writes to console no sooner than when we have the data back from the DB.
The suggested answer seems only to add yet another function?