Getting mysql:"Too many connections" despite using createPool in NodeJs - mysql

I am using Observables to display real-time data in Angular2+ application and the back-end is written in NodeJs that uses MySQL database. Since I am using Observables, I need tens of millions of connections of MySQL to keep the real-time work going.
But it is not possible to acquire so many connections. So I used pooling where a connection is created from the pool of connections. However, I fail to implement it. I still get the error:
Error: ER_CON_COUNT_ERROR: Too many connections"
How can I close my connections so that the connections are not outnumbered?
Front end code:
angular.component.ts
Observable.interval(10000).subscribe(x => {
this.viewData(Val);
// more functions
console.log(" Observable")
});
NodeJS code:
dashboard.service.js
function viewData(data) {
var sqlQuery = `
select * from TRANSACTION_PAYLOAD where INTERFACE_NAME = 'Highmark' AND (STATUS ='SUCCESS_RESPONSE')`
var deferred = Q.defer();
console.log("INSIDE NODE JS SERVICE");
var host = config.host;
var user = config.user;
var password = config.password;
var database = config.database;
var con = mysql.createPool({
host: host,
user: user,
password: password,
database: database
});
con.getConnection(function (err) {
console.log("Inside .getConnection ")
if (err) deferred.reject(err.name + ': ' + err.message);
con.query(sqlQuery,
function (err, result, fields) {
if (err) deferred.reject(err.name + ': ' + err.message);
console.log(result);
deferred.resolve(result);
});
});
return deferred.promise;
con.close();
}

you are creating mysql pool for every request. You supposed to have single connection pool.
connection pool manages connections made to mysql server automatically.
you need to take out connection pool initialization out of your function.
take the connection pull out
const mysqlConOptions = {
host: host,
user: user,
password: password,
database: database
};
var conPool = mysql.createPool(mysqlConOptions);
inside your function,
conn = await conPool.getConnection();

Related

How do I avoid max connections error in mysql?

This happens pretty frequently (once a week for about 30-40 minutes), where all of a sudden my database mentions max connections when I try to connect via heidisql, and any apis calls respond with the following error:
Cannot read property 'release' of undefined
I am calling .release() after every query in mysql. Is there something I am missing, am I suppose to call .end as well? I am using nodejs with mysql.
Here is the way I wrap every query and the pool code:
var mysql = require('mysql');
var mysql_pool = mysql.createPool({
connectionLimit: config.mysql.limit,
host: config.mysql.host,
user: config.mysql.user,
password: config.mysql.pw,
database: config.mysql.db //,
// debug: true
});
var qSelect = "SELECT id FROM Users";
var qValues = [];
var qCall = mysql.format(qSelect, qValues);
mysql_pool.getConnection(function(err_pool, connection) {
if (err_pool) {
connection.release();
console.log(' Error getting mysql_pool connection: ' + err_pool);
throw err_pool;
}
connection.query(qCall, function(err, userFound, fields) {
connection.release();
if (err) {
console.log("get user : " + err);
} else {
//some code here
}
});
Can someone please advise, appreciate it.
You should remove first connection.release() used in if loop
if (err_pool) {
console.log(' Error getting mysql_pool connection: ' + err_pool);
throw err_pool;
}

Connection to mysql works fine locally but not as AWS lambda function

I've created a simple mySQL database that I'm trying to access data from via an AWS Lambda function.
This is a version of the code that runs fine locally:
var mysql = require('mysql');
var config = require('./config.json');
var pool = mysql.createPool({
host : config.dbhost,
user : config.dbuser,
password : config.dbpassword,
database : config.dbname
});
pool.getConnection(function(err, connection) {
// Use the connection
connection.query("SELECT username FROM ClimbingDB.users WHERE email = 'testemail1'", function (error, results, fields) {
// And done with the connection.
connection.release();
// Handle error after the release.
if (error) throw error;
console.log(results);
process.exit();
});
});
This is that code converted to work with AWS Lambda:
var mysql = require('mysql');
var config = require('./config.json');
var pool = mysql.createPool({
host : config.dbhost,
user : config.dbuser,
password : config.dbpassword,
database : config.dbname
});
exports.handler = (event, context, callback) => {
//prevent timeout from waiting event loop
context.callbackWaitsForEmptyEventLoop = false;
pool.getConnection(function(err, connection) {
if (err) return callback(err)
// Use the connection
connection.query("SELECT username FROM ClimbingDB.users WHERE email = 'testemail1'", function (error, results, fields) {
// And done with the connection.
connection.release();
// Handle error after the release.
if (error) return callback(error);
else return callback(null,results);
});
});
};
Which times out with this error message:
{
"errorMessage": "2019-07-19T17:49:04.110Z 2f3e208c-62a6-4e90-b8ec-29398780a2a6 Task timed out after 3.00 seconds"
}
I'm not sure why it doesnt seem to be able to connect. I tried adding the function to a vpc and a security group that has access to RDB's, neither of which do anything. I'm not sure what I'm doing wrong here.
You will need:
The Amazon RDS instance in the same VPC as the AWS Lambda function
A security group on the Lambda function (Lambda-SG)
A security group on the RDS instance (DB-SG) that permits inbound connections on port 3306 from Lambda-SG
That is, DB-SG should specifically reference Lambda-SG (it will turn into a security group ID in the format sg-1234).
You might also want to increase the timeout of the Lambda function to give it a bit more time to run.

How to return a promise as well as close the mysql connection in the same code in NodeJS

We want our function to return a promise as well as close the database connection. We put the promise in try and connection closing command in finally to which it gives the following errors -
conPool.close() //It says Error - conPool.close() is not a function
conPool.release() // Error - conPool.release() is not a function
conPool.releaseConnection(); // Error - Cannot read property '_pool'
of undefined
Please help me close the mySQL connection. I want to return my data . But after return nothing works so the connection doesn't close. I am afraid i may loose out on maximum limit. I have even set maximum connections in mySQL but still. And there is a possible solution that i may declare conPool outside the function and make all the functions use this single conPool but this is also not working.
code -
function viewChoice() {
var sqlQuery;
sqlQuery = "select * from TRANSACTION_PAYLOAD where INTERFACE_NAME
= 'Vehicle RC' AND (STATUS ='INPUT_ERROR' OR STATUS ='ERROR')";
}
var deferred = Q.defer();
var host = config.host;
var user = config.user;
var password = config.password;
var database = config.database;
var conPool = mysql.createPool({
host: host,
user: user,
password: password,
database: database
});
try{
conPool.getConnection(function (err) {
console.log("Inside getConnection ")
if (err)
deferred.reject(err.name + ': ' + err.message);
conPool.query(sqlQuery,
function (err, result, fields) {
if (err) deferred.reject(err.name + ': ' + err.message);
console.log(result);
deferred.resolve(result);
});
});
return deferred.promise;
}
finally{
console.log("inside Finally")
conPool.releaseConnection();
}
}
From MySQL connector docs :
Closing all the connections in a pool
When you are done using the pool, you have to end all the connections or the Node.js event loop will stay active until the connections are closed by the MySQL server. This is typically done if the pool is used in a script or when trying to gracefully shutdown a server. To end all the connections in the pool, use the end method on the pool:
pool.end(function (err) {
// all connections in the pool have ended
});
The end method takes an optional callback that you can use to know when all the connections are ended.
Once pool.end is called, pool.getConnection and other operations can no longer be performed. Wait until all connections in the pool are released before calling pool.end. If you use the shortcut method pool.query, in place of pool.getConnection → connection.query → connection.release, wait until it completes.
pool.end calls connection.end on every active connection in the pool. This queues a QUIT packet on the connection and sets a flag to prevent pool.getConnection from creating new connections. All commands / queries already in progress will complete, but new commands won't execute.
Plus you may move your return deferred.promise; inside finally block

NodeJs - How to share MySQL pool accross my models to avoid 'ER_CON_COUNT_ERROR'

I'm currently testing my node app using ApacheBench. I run into an issue with my database which is ER_CON_COUNT_ERROR: Too many connections.
I'm using a short library on the top of MySQL node module that you can see just below
var mysql = require('mysql');
var config = require('path/to/config');
var message = require('./myMessageLib.js');
var pool = mysql.createPool({
connectionLimit : 100,
host: config.db.mysql.host,
user: config.db.mysql.user,
password: config.db.mysql.password,
database: config.db.mysql.database
});
var query = function(query_str, values, next) {
pool.getConnection((err, connection) => {
if (err) {
console.error("MySQL Fail to get a connection in pool : " + err);
if (typeof connection !== "undefined")
connection.release();
next(error, null);
return ;
}
connection.query(query_str, values, function(error, data, fields) {
connection.release();
if (error)
if (config.app.env.dev)
throw (error);
else {
next(error, null);
return (message.error("MySQL query failed : " + query_str + " / err : " + error));
}
if (data.length == 0)
next(null);
else
next(data);
})
})
}
exports.query = query;
I use this library in my model by doing something like this
var mysql = require('path/to/mysqllib');
/**
* Class PlayerModel
*/
function PlayerModel() { };
PlayerModel.prototype.get = function(id, next) {
mysql.query("SELECT ....", [id], function(player) {
// stuff
})
}
module.exports = PlayerModel;
The things is on my homepage I use different models like the one presented above and each one launch a query to get some database information. When I launch an ApacheBench with only 50 concurrency levels I got the ER_CON_COUNT_ERROR: Too many connections. So I've got the feeling that the pool isn't well made because it seems that it didn't respect the connections limit of 100 written in the short MySQL lib.
I was thinking about creating and storing the pool in the global nodejs variable to be able to share it correctly accros my modules but I'm not sure it's a good way and maybe also I'm doing something wrong on my pool implentation.
Do you have any idea or improvements to suggest ?
Thanks mates!
I figured out the issue.
My app was deploying in cluster mode. Two process were running at the same time. Because of that, two pools of 100 connections could have been created which is resulting on a total of 200 connections which is higher than the MySQL default connection limit.
Great that found a solution and here's another one with less code.
create a js file, dbconnection.js for example
var mysql = require("mysql");
var pool = mysql.createPool({
connectionLimit: 10,
host: '...',
user: '...',
password: '...',
database: '...',
dateStrings: true
});
exports.connection = {
query: function () {
var queryArgs = Array.prototype.slice.call(arguments),
events = [],
eventNameIndex = {};
pool.getConnection(function (err, conn) {
if (err) {
if (eventNameIndex.error) {
eventNameIndex.error();
}
}
if (conn) {
var q = conn.query.apply(conn, queryArgs);
q.on('end', function () {
conn.release();
});
events.forEach(function (args) {
q.on.apply(q, args);
});
}
});
return {
on: function (eventName, callback) {
events.push(Array.prototype.slice.call(arguments));
eventNameIndex[eventName] = callback;
return this;
}
};
}
};
In the other file where you want to use the connection
var db = require('./dbconnection.js');
And instead of
connection.query
Use
db.connection.query

Received invalid field length error in nodejs with mysql application

I'm getting below error while fetching records (apx 50 rows) from my sql database. My application is developed in nodejs with express.
var common = require(__base + 'routes/common.js');
var dbhelper = require(__base + 'routes/dbhelper.js');
exports.GetStates = function (callback) {
dbhelper.pool.getConnection(function (err, connection) {
// Use the connection
connection.query('CALL GetStates()',
function (err, res) {
connection.release();
if (err) {
common.ActionOutput.Status = common.ActionStatus.Error;
common.ActionOutput.Message = 'System Error: ' + err.message;
} else {
common.ActionOutput.Status = common.ActionStatus.Success;
common.ActionOutput.Result = res[0][0];
}
return callback(JSON.stringify(common.ActionOutput));
});
});
};
dbhelper.js is
// Database connection
var mysql = require('mysql');
var pool = mysql.createPool({
connectionLimit: 10,
host: 'my ip',
user: 'user',
password: 'pass',,
database: 'ssdsdas'
});
exports.pool = pool;
I ran into the same error message but the causes of our issues might be different. Ultimately, the cause of my issue is that the column names in my stored procedure did not have back ticks (`) resulting in them being treated as variables.
Adding back ticks in my stored procedure body for column names fixed it.