AWS Lambda nodejs mysql query inside loop is not working - mysql

module.exports.handler = async (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
let index = 0;
var client = mysql.createConnection({
host: process.env.rds_host,
user: process.env.rds_username,
password: process.env.rds_password,
port: process.env.rds_port
});
client.connect((err) => {
if (err) throw err;
console.log("Connected!");
});
let array = [];
let queries = ["query1", "query2", "query3", "query4"];
queries.map(q => {
array.push(getQueryResult(client, q));
});
Promise.all(array).then(result => {
console.log(result);
});
callback(null, {});
};
const getQueryResult = async (client, query) => {
return new Promise((resolve, reject) => {
client.query(query, function (err, result) {
if (err) {
reject(err);
}
resolve(result);
});
});
};
Above is my lambda scripts to execute multiple query from mysql. The problem is I didn't get any result and error message from above scripts. Please help me something is missing inside my scripts?

The issue is >> code is not waiting to finish Promise
You can resolve by :
Add callback in then
Promise.all(array).then(result => {
console.log(result);
callback(null, {});
});
OR
Use await
let result = await Promise.all(promiseArray);
console.log(result)
callback(null, {});
Note: Use try-catch to handle error in for await
Also, don't use map to loop array instead use For loop.

There are two or three (potential) issues with your code above:
Your if statement does not get evaluated because of the typeof client predicate does not return true.
your mysql port conflicts with your localhost port (assumption)
Change your if block as such:
// check if `dotenv` has been called if you use it
require('dotenv').config();
// access the state property on mysql object
// if you have closed your connection previously
// this will get evaluated by the parser
if (mysql.state === "disconnected") {
var client = mysql.createConnection({
host: process.env.rds_host,
user: process.env.rds_username,
password: process.env.rds_password,
port: process.env.rds_port // is your port the same as your localhost?
// include your database name here
});
// I suggest you to await your client connection
// since mysql executes sequentially, not asynchronously
client.connect(function(err) => {
if (err) {
console.error('error connecting: ' + err.stack);
return;
}
console.log("Connected!");
});
}
if the error persists, it means that your enviroment variables are not set correctly so your db configuration should be reviewed (see inline comments).

Related

Node exports MySQL Pool Cluster

I'm new to node and I'm trying to use a MySQL pool cluster in but I'm not exactly sure how to export it.
At the moment I have the following in /libs/mysql.js:
poolCluster.add('db1', {
host: config.databases.hostname,
user: config.databases.db1.username,
password: config.databases.db1.password,
database: config.databases.db1.database,
connectionLimit: config.databases.connectionLimit
});
poolCluster.add('db2', {
host: config.databases.hostname,
user: config.databases.db2.username,
password: config.databases.db2.password,
database: config.databases.db2.database,
connectionLimit: config.databases.connectionLimit
});
module.exports = {
getConnection: (callback) => {
return poolCluster.getConnection(callback);
}
};
I'm trying to use it in models/monitor.js as below:
let poolCluster = require('../libs/mysql');
let moment = require('moment');
exports.select = function (sql, values, callback) {
poolCluster.getConnection('db1', (err, connection) => {
if (err) {
callback(err);
} else {
connection.query(sql, values, (err, result) => {
connection.release();
if (err) {
console.log(err);
callback(err);
} else {
callback(null, result)
}
})
}
})
};
The issue now is I'm getting an error stating cb is not a function.
Is this the correct way to export a mysql pool cluster in node?
You are exporting as getConnection: (callback) => {} but you are calling the same function with getConnection(string, callback).
According to the doc:
You can call getConnection like:
// Target Group : ALL(anonymous, MASTER, SLAVE1-2), Selector : round-robin(default)
poolCluster.getConnection(function (err, connection) {});
// Target Group : MASTER, Selector : round-robin
poolCluster.getConnection('MASTER', function (err, connection) {});
So, basically you need to pass the arguments you are getting from YOUR getConnection function to mysql's getConnection function. So this should do the trick:
module.exports = {
getConnection: (...args) => {
return poolCluster.getConnection(...args);
}
};

Make synchronous MySQL queries in NodeJS

I have been doing google searches for 5 days, I hope to find the solution ... I know that it does not work because it is asynchronous, but I need the program (it is a Discord bot) to respond with a data that I get from a DB. I have tried Promises and callbacks, but I do not know if it is because I am a novice with asynchronous, that nothing works for me.
const con = mysql.createConnection({
host: datos.host,
user: datos.user,
password: datos.password,
database: datos.database
});
function leerPromesa() {
var promise = new Promise(function (resolve, reject) {
con.query('SELECT * from ranking;', function (err, rows, fields) {
if (err) {
reject(err);
return
}
resolve(rows);
rows.forEach(element => console.log(element));
})
});
return promise;
};
var promesa = leerPromesa();
promesa.then(
function (rows) {
rows.forEach(element => msg.reply(element));
},
function (err) {
msg.reply(err);
}
);
con.end();
What the bot does is respond with blank text.
First, you're not really connecting to database.
If you refer to docs https://github.com/mysqljs/mysql:
var connection = mysql.createConnection({
host : 'localhost',
user : 'me',
password : 'secret',
database : 'my_db'
});
// then connect method
connection.connect();
So your code will never work..
Second, you are closing connection before any query execution:
con.end();
Correct is to close connection after leerPromesa function execution.
Finally, code could look something like this:
const con = mysql.createConnection({
host: datos.host,
user: datos.user,
password: datos.password,
database: datos.database
});
con.connect();
function leerPromesa() {
return new Promise(function(resolve, reject) {
con.query("SELECT * from ranking;", function(err, rows, fields) {
if (err) {
return reject(err);
}
return resolve(rows);
});
});
}
leerPromesa()
.then(
function(rows) {
rows.forEach(element => msg.reply(element));
},
function(err) {
msg.reply(err);
}
)
.finally(function() {
con.end();
});
I used finally method on Promise to close connection in every situation https://developer.mozilla.org/ru/docs/Web/JavaScript/Reference/Global_Objects/Promise/finally

Serverless with nodejs and mysql => Process exited before completing request

I'm developing some functions with serverless with the nodejs template. I have a service that connects to a mysql database and retrieves some data. Everything is fine when I make the first call, but when I repeat it I receive a "Process exited before completing request" error message.
If I try that same call again, I receive data. So the service is doing right on the odd calls and it's returning the error on the even calls (funny right?). This is the code of the handler function:
module.exports.getAll = (event, context, done) => {
deviceRepository.getAllDevices().then((response) => {
done(null, { response });
}).catch((error) => {
done(error);
});
};
and this is the code of the repository function:
const mysql = require('mysql');
const when = require('when');
const config = require('./config');
const conn = mysql.createConnection({
host: config.RDSHOST,
user: config.RDSUSER,
password: config.RDSPASS,
database: config.RDSDB,
port: config.RDSPORT
});
module.exports.getAllDevices = () => {
const deferred = when.defer();
conn.connect();
conn.query('SELECT * FROM device', (err, rows) => {
if (err) {
deferred.reject(err);
} else {
deferred.resolve(rows);
}
conn.end();
});
return deferred.promise;
};
As you can see I use promises with the 'when' library. I call the 'done' callback in the handler, and there should be a response from the promise in every possible situation.
I can't see what is wrong with this and why is making the odd requests wrong. Anyone can help?
Thanks in advance!
Solved by myself...
The problem is that I was making the createConnection outside of the handler (when I declared the conn constant).
Moving the createConnection declaration inside the handler function works as expected in every call.
const mysql = require('mysql');
const when = require('when');
const config = require('./config');
module.exports.getAllDevices = () => {
const conn = mysql.createConnection({
host: config.RDSHOST,
user: config.RDSUSER,
password: config.RDSPASS,
database: config.RDSDB,
port: config.RDSPORT
});
const deferred = when.defer();
conn.connect();
conn.query('SELECT * FROM device', (err, rows) => {
if (err) {
deferred.reject(err);
} else {
deferred.resolve(rows);
}
conn.end();
});
return deferred.promise;
};
Hope this helps. Thanks!

node.js + mysql connection pooling

I'm trying to figure out how to structure my application to use MySQL most efficent way. I'm using node-mysql module. Other threads here suggested to use connection pooling so i set up a little module mysql.js
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'localhost',
user : 'root',
password : 'root',
database : 'guess'
});
exports.pool = pool;
Now whenever I want to query mysql I require this module and then query the databse
var mysql = require('../db/mysql').pool;
var test = function(req, res) {
mysql.getConnection(function(err, conn){
conn.query("select * from users", function(err, rows) {
res.json(rows);
})
})
}
Is this good approach? I couldn't really find too much examples of using mysql connections besides very simple one where everything is done in main app.js script so I don't really know what the convention / best practices are.
Should I always use connection.end() after each query? What if I forget about it somewhere?
How to rewrite the exports part of my mysql module to return just a connection so I don't have to write getConnection() every time?
It's a good approach.
If you just want to get a connection add the following code to your module where the pool is in:
var getConnection = function(callback) {
pool.getConnection(function(err, connection) {
callback(err, connection);
});
};
module.exports = getConnection;
You still have to write getConnection every time. But you could save the connection in the module the first time you get it.
Don't forget to end the connection when you are done using it:
connection.release();
You should avoid using pool.getConnection() if you can. If you call pool.getConnection(), you must call connection.release() when you are done using the connection. Otherwise, your application will get stuck waiting forever for connections to be returned to the pool once you hit the connection limit.
For simple queries, you can use pool.query(). This shorthand will automatically call connection.release() for you—even in error conditions.
function doSomething(cb) {
pool.query('SELECT 2*2 "value"', (ex, rows) => {
if (ex) {
cb(ex);
} else {
cb(null, rows[0].value);
}
});
}
However, in some cases you must use pool.getConnection(). These cases include:
Making multiple queries within a transaction.
Sharing data objects such as temporary tables between subsequent queries.
If you must use pool.getConnection(), ensure you call connection.release() using a pattern similar to below:
function doSomething(cb) {
pool.getConnection((ex, connection) => {
if (ex) {
cb(ex);
} else {
// Ensure that any call to cb releases the connection
// by wrapping it.
cb = (cb => {
return function () {
connection.release();
cb.apply(this, arguments);
};
})(cb);
connection.beginTransaction(ex => {
if (ex) {
cb(ex);
} else {
connection.query('INSERT INTO table1 ("value") VALUES (\'my value\');', ex => {
if (ex) {
cb(ex);
} else {
connection.query('INSERT INTO table2 ("value") VALUES (\'my other value\')', ex => {
if (ex) {
cb(ex);
} else {
connection.commit(ex => {
cb(ex);
});
}
});
}
});
}
});
}
});
}
I personally prefer to use Promises and the useAsync() pattern. This pattern combined with async/await makes it a lot harder to accidentally forget to release() the connection because it turns your lexical scoping into an automatic call to .release():
async function usePooledConnectionAsync(actionAsync) {
const connection = await new Promise((resolve, reject) => {
pool.getConnection((ex, connection) => {
if (ex) {
reject(ex);
} else {
resolve(connection);
}
});
});
try {
return await actionAsync(connection);
} finally {
connection.release();
}
}
async function doSomethingElse() {
// Usage example:
const result = await usePooledConnectionAsync(async connection => {
const rows = await new Promise((resolve, reject) => {
connection.query('SELECT 2*4 "value"', (ex, rows) => {
if (ex) {
reject(ex);
} else {
resolve(rows);
}
});
});
return rows[0].value;
});
console.log(`result=${result}`);
}
You will find this wrapper usefull :)
var pool = mysql.createPool(config.db);
exports.connection = {
query: function () {
var queryArgs = Array.prototype.slice.call(arguments),
events = [],
eventNameIndex = {};
pool.getConnection(function (err, conn) {
if (err) {
if (eventNameIndex.error) {
eventNameIndex.error();
}
}
if (conn) {
var q = conn.query.apply(conn, queryArgs);
q.on('end', function () {
conn.release();
});
events.forEach(function (args) {
q.on.apply(q, args);
});
}
});
return {
on: function (eventName, callback) {
events.push(Array.prototype.slice.call(arguments));
eventNameIndex[eventName] = callback;
return this;
}
};
}
};
Require it, use it like this:
db.connection.query("SELECT * FROM `table` WHERE `id` = ? ", row_id)
.on('result', function (row) {
setData(row);
})
.on('error', function (err) {
callback({error: true, err: err});
});
I am using this base class connection with mysql:
"base.js"
var mysql = require("mysql");
var pool = mysql.createPool({
connectionLimit : 10,
host: Config.appSettings().database.host,
user: Config.appSettings().database.username,
password: Config.appSettings().database.password,
database: Config.appSettings().database.database
});
var DB = (function () {
function _query(query, params, callback) {
pool.getConnection(function (err, connection) {
if (err) {
connection.release();
callback(null, err);
throw err;
}
connection.query(query, params, function (err, rows) {
connection.release();
if (!err) {
callback(rows);
}
else {
callback(null, err);
}
});
connection.on('error', function (err) {
connection.release();
callback(null, err);
throw err;
});
});
};
return {
query: _query
};
})();
module.exports = DB;
Just use it like that:
var DB = require('../dal/base.js');
DB.query("select * from tasks", null, function (data, error) {
callback(data, error);
});
When you are done with a connection, just call connection.release() and the connection will return to the pool, ready to be used again by someone else.
var mysql = require('mysql');
var pool = mysql.createPool(...);
pool.getConnection(function(err, connection) {
// Use the connection
connection.query('SELECT something FROM sometable', function (error, results, fields) {
// And done with the connection.
connection.release();
// Handle error after the release.
if (error) throw error;
// Don't use the connection here, it has been returned to the pool.
});
});
If you would like to close the connection and remove it from the pool, use connection.destroy() instead. The pool will create a new connection the next time one is needed.
Source: https://github.com/mysqljs/mysql
You can use this format as I used
const mysql = require('mysql');
const { HOST, USERNAME, PASSWORD, DBNAME, PORT } = process.env;
console.log();
const conn = mysql.createPool({
host: HOST,
user: USERNAME,
password: PASSWORD,
database: DBNAME
}, { debug: true });
conn.query('SELECT 1 + 1 AS solution', function (error, results, fields) {
if (error) throw error;
console.log('Db is connected - The solution is: ', results[0].solution);
});
module.exports = conn;
Using the standard mysql.createPool(), connections are lazily created by the pool. If you configure the pool to allow up to 100 connections, but only ever use 5 simultaneously, only 5 connections will be made. However if you configure it for 500 connections and use all 500 they will remain open for the durations of the process, even if they are idle!
This means if your MySQL Server max_connections is 510 your system will only have 10 mySQL connections available until your MySQL Server closes them (depends on what you have set your wait_timeout to) or your application closes! The only way to free them up is to manually close the connections via the pool instance or close the pool.
mysql-connection-pool-manager module was created to fix this issue and automatically scale the number of connections dependant on the load. Inactive connections are closed and idle connection pools are eventually closed if there has not been any activity.
// Load modules
const PoolManager = require('mysql-connection-pool-manager');
// Options
const options = {
...example settings
}
// Initialising the instance
const mySQL = PoolManager(options);
// Accessing mySQL directly
var connection = mySQL.raw.createConnection({
host : 'localhost',
user : 'me',
password : 'secret',
database : 'my_db'
});
// Initialising connection
connection.connect();
// Performing query
connection.query('SELECT 1 + 1 AS solution', function (error, results, fields) {
if (error) throw error;
console.log('The solution is: ', results[0].solution);
});
// Ending connection
connection.end();
Ref: https://www.npmjs.com/package/mysql-connection-pool-manager
i always use connection.relase(); after pool.getconnetion like
pool.getConnection(function (err, connection) {
connection.release();
if (!err)
{
console.log('*** Mysql Connection established with ', config.database, ' and connected as id ' + connection.threadId);
//CHECKING USERNAME EXISTENCE
email = receivedValues.email
connection.query('SELECT * FROM users WHERE email = ?', [email],
function (err, rows) {
if (!err)
{
if (rows.length == 1)
{
if (bcrypt.compareSync(req.body.password, rows[0].password))
{
var alldata = rows;
var userid = rows[0].id;
var tokendata = (receivedValues, userid);
var token = jwt.sign(receivedValues, config.secret, {
expiresIn: 1440 * 60 * 30 // expires in 1440 minutes
});
console.log("*** Authorised User");
res.json({
"code": 200,
"status": "Success",
"token": token,
"userData": alldata,
"message": "Authorised User!"
});
logger.info('url=', URL.url, 'Responce=', 'User Signin, username', req.body.email, 'User Id=', rows[0].id);
return;
}
else
{
console.log("*** Redirecting: Unauthorised User");
res.json({"code": 200, "status": "Fail", "message": "Unauthorised User!"});
logger.error('*** Redirecting: Unauthorised User');
return;
}
}
else
{
console.error("*** Redirecting: No User found with provided name");
res.json({
"code": 200,
"status": "Error",
"message": "No User found with provided name"
});
logger.error('url=', URL.url, 'No User found with provided name');
return;
}
}
else
{
console.log("*** Redirecting: Error for selecting user");
res.json({"code": 200, "status": "Error", "message": "Error for selecting user"});
logger.error('url=', URL.url, 'Error for selecting user', req.body.email);
return;
}
});
connection.on('error', function (err) {
console.log('*** Redirecting: Error Creating User...');
res.json({"code": 200, "status": "Error", "message": "Error Checking Username Duplicate"});
return;
});
}
else
{
Errors.Connection_Error(res);
}
});

Node.js MySQL Needing Persistent Connection

I need a persistent MySQL connection for my Node web app. The problem is that this happens about a few times a day:
Error: Connection lost: The server closed the connection.
at Protocol.end (/var/www/n/node_modules/mysql/lib/protocol/Protocol.js:73:13)
at Socket.onend (stream.js:79:10)
at Socket.EventEmitter.emit (events.js:117:20)
at _stream_readable.js:895:16
at process._tickCallback (node.js:415:13)
error: Forever detected script exited with code: 8
error: Forever restarting script for 2 time
info: socket.io started
Here is my connection code:
// Yes I know multipleStatements can be dangerous in the wrong hands.
var sql = mysql.createConnection({
host: 'localhost',
user: 'my_username',
password: 'my_password',
database: 'my_database',
multipleStatements: true
});
sql.connect();
function handleDisconnect(connection) {
connection.on('error', function(err) {
if (!err.fatal) {
return;
}
if (err.code !== 'PROTOCOL_CONNECTION_LOST') {
throw err;
}
console.log('Re-connecting lost connection: ' + err.stack);
sql = mysql.createConnection(connection.config);
handleDisconnect(sql);
sql.connect();
});
}
handleDisconnect(sql);
As you can see, the handleDisconnect code does not work..
Use the mysql connection pool. It will reconnect when a connection dies and you get the added benefit of being able to make multiple sql queries at the same time. If you don't use the database pool, your app will block database requests while waiting for currently running database requests to finish.
I usually define a database module where I keep my queries separate from my routes. It looks something like this...
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'example.org',
user : 'bob',
password : 'secret'
});
exports.getUsers = function(callback) {
pool.getConnection(function(err, connection) {
if(err) {
console.log(err);
callback(true);
return;
}
var sql = "SELECT id,name FROM users";
connection.query(sql, [], function(err, results) {
connection.release(); // always put connection back in pool after last query
if(err) {
console.log(err);
callback(true);
return;
}
callback(false, results);
});
});
});
I know this is super delayed, but I've written a solution to this that I think might be a bit more generic and usable. I had written an app entirely dependent on connection.query() and switching to a pool broke those calls.
Here's my solution:
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'localhost',
user : 'user',
password : 'secret',
database : 'test',
port : 3306
});
module.exports = {
query: function(){
var sql_args = [];
var args = [];
for(var i=0; i<arguments.length; i++){
args.push(arguments[i]);
}
var callback = args[args.length-1]; //last arg is callback
pool.getConnection(function(err, connection) {
if(err) {
console.log(err);
return callback(err);
}
if(args.length > 2){
sql_args = args[1];
}
connection.query(args[0], sql_args, function(err, results) {
connection.release(); // always put connection back in pool after last query
if(err){
console.log(err);
return callback(err);
}
callback(null, results);
});
});
}
};
This instantiates the pool once, then exports a method named query. Now, when connection.query() is called anywhere, it calls this method, which first grabs a connection from the pool, then passes the arguments to the connection. It has the added effect of grabbing the callback first, so it can callback any errors in grabbing a connection from the pool.
To use this, simply require it as module in place of mysql. Example:
var connection = require('../middleware/db');
function get_active_sessions(){
connection.query('Select * from `sessions` where `Active`=1 and Expires>?;', [~~(new Date()/1000)], function(err, results){
if(err){
console.log(err);
}
else{
console.log(results);
}
});
}
This looks just like the normal query, but actually opens a pool and grabs a connection from the pool in the background.
In response to #gladsocc question:
Is there a way to use pools without refactoring everything? I have
dozens of SQL queries in the app.
This is what I ended up building. It's a wrapper for the query function. It will grab the connection, do the query, then release the connection.
var pool = mysql.createPool(config.db);
exports.connection = {
query: function () {
var queryArgs = Array.prototype.slice.call(arguments),
events = [],
eventNameIndex = {};
pool.getConnection(function (err, conn) {
if (err) {
if (eventNameIndex.error) {
eventNameIndex.error();
}
}
if (conn) {
var q = conn.query.apply(conn, queryArgs);
q.on('end', function () {
conn.release();
});
events.forEach(function (args) {
q.on.apply(q, args);
});
}
});
return {
on: function (eventName, callback) {
events.push(Array.prototype.slice.call(arguments));
eventNameIndex[eventName] = callback;
return this;
}
};
}
};
And I use it like I would normally.
db.connection.query("SELECT * FROM `table` WHERE `id` = ? ", row_id)
.on('result', function (row) {
setData(row);
})
.on('error', function (err) {
callback({error: true, err: err});
});