AWS RDS MySQL Connection count - mysql

I have a simple program that connects to an AWS MySQL DB via Lambda.
Every request from the app is basically a single query on the database. So if I have 3 users using the app, it would be impossible for more than 3 queries to be run on the database at the same time.
Yet when I look at my connection count it could be as high as 15. I've been having trouble understanding exactly how this should work.
If the code is written correctly does that mean if there's 3 users at once, I shouldn't have more than 3 connections?
I've been trying to find a basic article to explain this but obviously haven't hence I'm here.
Thank you.
EDITED TO ADD CODE
Below is a cut down version of my code
var mysql = require('mysql');
var pool =[];
exports.handler = (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
pool = mysql.createPool({
host : 'host-details',
user : 'username',
password : 'password',
database : 'db'
});
return myFunc(event, context, callback, data.Username);
}
};
const myFunc = async (event, context, callback) => {
const query = 'SELECT * from db.table';
const data = await dbQueryAsyncWL(context, callback, query);
return callback(null, {data: data});
}
const dbQueryAsyncWL = async (context, callback, query) => {
return new Promise((resolve) => {
context.callbackWaitsForEmptyEventLoop = false;
pool.getConnection(function(error, connection) {
connection.query(query, function (error, results, fields) {
connection.release();
if (error) {callback(error);} else {resolve(results);}
});
});
})
}

pool = mysql.createPool({
host : 'host-details',
user : 'username',
password : 'password',
database : 'db'
});
Check the size / minimum size of the connection pool.
What is the connection pool configuration, like idleTime etc before closing a connection.
What trends are you noticing around the size of connection pool, is it stable or increase/decreasing ?

Looks like my issue was that I was creating the pool inside the exports.handler. Once I moved it outside... well a picture says it all.
https://i.ibb.co/KDvfdhp/Screen-Shot-2020-06-29-at-10-26-49-pm.png

Related

Why might this node.js mysql query be preventing my program from ending?

I am trying to fix a bug in my node.js application, which is preventing a program I have written from ending successfully.
The script is returning the required results, but something is still running in the background which appears to be preventing it from terminating.
Through process of elimination, I have been able to narrow down the culprit to this getPlayers() function; but I am not sure why the problem is being caused.
Firstly, I am exporting the database pool from a module, like so:
require('dotenv').config()
const mysql = require("mysql"),
pool = mysql.createPool({
host: process.env.dbHost,
user: process.env.dbUser,
password: process.env.dbPassword,
database: process.env.dbDatabase
})
pool.connectionLimit = 15
module.exports = pool
And this is the query that appears to be causing all of the problems:
const getPlayers = () => {
return new Promise((resolve, reject) => {
db.query('SELECT * FROM cc_players', (err, res) => {
if(err)
console.log(err)
resolve(res)
})
})
}
If I comment out the db.query() function, and simply resolve an empty array, the script terminates as expected. Is there something about the database query that could be causing the script to continue running in the background?
Because you're creating a pool, previously opened SQL connections will not be closed, and instead kept for later uses. This is why Node.JS never exits.
To fix this, the mysql package provides a pool.end function to close all connections in the pool. You should call it when your script is ready to exit, perhaps like so:
function onExit() {
// ...
// Assuming ``db`` is the pool you created
db.end()
// ...
}
Beware that no further SQL operations can be performed on the pool after pool.end is called.
You created a pool, which will keep connections open for re-use. As long as there's open sockets, Node.js will not exit.
You have to use alternative way because create a pool will keep the connection opened in background, so you can use createConnection function instead of createPool then you can close the connection manually
let mysql = require('mysql');
let connection = mysql.createConnection({
host: process.env.dbHost,
user: process.env.dbUser,
password: process.env.dbPassword,
database: process.env.dbDatabase
});
const getPlayers = () => {
return new Promise((resolve, reject) => {
db.query('SELECT * FROM cc_players', (err, res) => {
if(err)
console.log(err)
resolve(res)
})
})
}
And once get the callback you can close the connection
getPlayer().then(res => {
.
.
.
connection.end();
})

Connection to mysql works fine locally but not as AWS lambda function

I've created a simple mySQL database that I'm trying to access data from via an AWS Lambda function.
This is a version of the code that runs fine locally:
var mysql = require('mysql');
var config = require('./config.json');
var pool = mysql.createPool({
host : config.dbhost,
user : config.dbuser,
password : config.dbpassword,
database : config.dbname
});
pool.getConnection(function(err, connection) {
// Use the connection
connection.query("SELECT username FROM ClimbingDB.users WHERE email = 'testemail1'", function (error, results, fields) {
// And done with the connection.
connection.release();
// Handle error after the release.
if (error) throw error;
console.log(results);
process.exit();
});
});
This is that code converted to work with AWS Lambda:
var mysql = require('mysql');
var config = require('./config.json');
var pool = mysql.createPool({
host : config.dbhost,
user : config.dbuser,
password : config.dbpassword,
database : config.dbname
});
exports.handler = (event, context, callback) => {
//prevent timeout from waiting event loop
context.callbackWaitsForEmptyEventLoop = false;
pool.getConnection(function(err, connection) {
if (err) return callback(err)
// Use the connection
connection.query("SELECT username FROM ClimbingDB.users WHERE email = 'testemail1'", function (error, results, fields) {
// And done with the connection.
connection.release();
// Handle error after the release.
if (error) return callback(error);
else return callback(null,results);
});
});
};
Which times out with this error message:
{
"errorMessage": "2019-07-19T17:49:04.110Z 2f3e208c-62a6-4e90-b8ec-29398780a2a6 Task timed out after 3.00 seconds"
}
I'm not sure why it doesnt seem to be able to connect. I tried adding the function to a vpc and a security group that has access to RDB's, neither of which do anything. I'm not sure what I'm doing wrong here.
You will need:
The Amazon RDS instance in the same VPC as the AWS Lambda function
A security group on the Lambda function (Lambda-SG)
A security group on the RDS instance (DB-SG) that permits inbound connections on port 3306 from Lambda-SG
That is, DB-SG should specifically reference Lambda-SG (it will turn into a security group ID in the format sg-1234).
You might also want to increase the timeout of the Lambda function to give it a bit more time to run.

discord.js/node.js make code wait until sql query returns result

I am working on a discord.js bot, and I'm storing a bunch of information on various servers in a database. The problem is, that the code doesn't wait for the database to return the results. In the current situation, I'm trying to check if the server specific prefix checks out.
I tried using async and await at various places, but those didn't work. If I could, I'd rather not use .then(), because I don't really want to put all the commands inside a .then().
const { Client, Attachment, RichEmbed } = require('discord.js');
const client = new Client();
const mysql = require("mysql");
const config = require("./config.json")
var con = mysql.createConnection({
host: 'localhost',
user: 'root',
password: '',
database: 'botdb'
})
client.on("ready", () => {
console.log("I'm ready")
})
client.on("message", message => {
if (message.author.bot) return;
if (message.channel.type === 'dm') return;
let msg = message.content.split(" ");
let command = msg[0];
let prefix;
con.query(`SELECT * FROM serversettings WHERE ServerID = ${message.guild.id}`, (err, rows) => {
if (err) throw err;
prefix = rows[0].Prefix;
console.log(prefix)
})
console.log(`Prefix: ${prefix}, Command: ${command}`)
if (command === `${prefix}examplecommand`) {
//Do something
}
//Other code that uses prefix and command
}
It should log the prefix first, and then the Prefix: ${prefix}, Command: ${command} part, but it does it the other way around, so the examplecommand doesn't work.
Your result is caused by the fact that what's outside your query callback is executed immediately after the call. Keep in mind the mysql module is callback-based.
Possible Solutions
Place the code inside the callback so it's executed when the query is completed.
Wrap the query in a promise and await it.
function getGuild(guildID) {
return new Promise((resolve, reject) => {
con.query(`SELECT * FROM serversettings WHERE ServerID = '${guildID}', (err, rows) => {
if (err) return reject(err);
resolve(rows);
});
});
}
const [guild] = await getGuild(message.guild.id) // destructuring 'rows' array
.catch(console.error);
console.log(guild.prefix);
Use a Promise-based version of a MySQL wrapper, like promise-mysql. You could use it the same way as the code above, without worrying about coding your own Promises.
const [guild] = await con.query(`SELECT * FROM serversettings WHERE serverID = '${message.guild.id}'`)
.catch(console.error);
console.log(guild.prefix);

Lambda NodeJS MySQL Task Timed out

I am trying to learn how to connect MySQL using lambda functions in AWS. I have followed a couple of instructions online and basically ended up with this code:
var mysql = require('mysql');
var pool = mysql.createPool({
connectionLimit : 1000,
connectTimeout : 60 * 60 * 1000,
acquireTimeout : 60 * 60 * 1000,
timeout : 60 * 60 * 1000,
host: "foo-bar-123.us-east-2.rds.amazonaws.com",
user: "root",
password: "pass123",
database: "sample_db",
});
exports.handler = (event, context, callback) => {
// prevent timeout from waiting event loop
context.callbackWaitsForEmptyEventLoop = false;
pool.getConnection(function(err, connection) {
if (err) throw err;
// Use the connection
connection.query('SELECT id FROM customer limit 10;', function (error, results, fields) {
// And done with the connection.
connection.release();
// Handle error after the release.
if (error) callback(error);
else callback(null,results);
});
});
};
This is working on my local but when I zip this code and uploaded it as a lambda function, this returns
Response:
{
"errorMessage": "2018-11-13T02:16:10.339Z 0562b432-e6ea-11e8-81ef-bd64aa1af0a4 Task timed out after 30.03 seconds"
}
It times out no matter how many seconds I set it to.
I have pretty much set everything at default since I am new to all of these but I have added AmazonRDSFullAccess to the role of lambda function.
Does anyone have any idea what may be wrong or missing in my setup?
Thanks.
After doing some trial and errors, I was able to make it work and what I was missing is that I did not allow All TCP in the inbound of my RDS Security group. After that, I set it as my lambda function to No VPC, and it was able to query properly.
This link: https://dzone.com/articles/querying-rds-mysql-db-with-nodejs-lambda-function and the stack overflow link posted in there (which is this: AWS Lambda RDS connection timeout) helped me a lot in figuring out what was wrong with my code/setup.
Here is the final code that I ended up using.
const mysql = require('mysql');
const pool = mysql.createPool({
host: "foo-bar-123.us-east-2.rds.amazonaws.com",
user: "root",
password: "pass123",
database: "sample_db"
});
exports.handler = (event, context, callback) => {
//prevent timeout from waiting event loop
context.callbackWaitsForEmptyEventLoop = false;
pool.getConnection((err, connection) => {
if(err) throw err;
// Use the connection
connection.query('SELECT id FROM customer limit 10;', (error, results, fields) => {
// And done with the connection.
connection.release();
// Handle error after the release.
if (error) callback(error);
else callback(null,results);
});
});
};
Thanks!

NodeJs - How to share MySQL pool accross my models to avoid 'ER_CON_COUNT_ERROR'

I'm currently testing my node app using ApacheBench. I run into an issue with my database which is ER_CON_COUNT_ERROR: Too many connections.
I'm using a short library on the top of MySQL node module that you can see just below
var mysql = require('mysql');
var config = require('path/to/config');
var message = require('./myMessageLib.js');
var pool = mysql.createPool({
connectionLimit : 100,
host: config.db.mysql.host,
user: config.db.mysql.user,
password: config.db.mysql.password,
database: config.db.mysql.database
});
var query = function(query_str, values, next) {
pool.getConnection((err, connection) => {
if (err) {
console.error("MySQL Fail to get a connection in pool : " + err);
if (typeof connection !== "undefined")
connection.release();
next(error, null);
return ;
}
connection.query(query_str, values, function(error, data, fields) {
connection.release();
if (error)
if (config.app.env.dev)
throw (error);
else {
next(error, null);
return (message.error("MySQL query failed : " + query_str + " / err : " + error));
}
if (data.length == 0)
next(null);
else
next(data);
})
})
}
exports.query = query;
I use this library in my model by doing something like this
var mysql = require('path/to/mysqllib');
/**
* Class PlayerModel
*/
function PlayerModel() { };
PlayerModel.prototype.get = function(id, next) {
mysql.query("SELECT ....", [id], function(player) {
// stuff
})
}
module.exports = PlayerModel;
The things is on my homepage I use different models like the one presented above and each one launch a query to get some database information. When I launch an ApacheBench with only 50 concurrency levels I got the ER_CON_COUNT_ERROR: Too many connections. So I've got the feeling that the pool isn't well made because it seems that it didn't respect the connections limit of 100 written in the short MySQL lib.
I was thinking about creating and storing the pool in the global nodejs variable to be able to share it correctly accros my modules but I'm not sure it's a good way and maybe also I'm doing something wrong on my pool implentation.
Do you have any idea or improvements to suggest ?
Thanks mates!
I figured out the issue.
My app was deploying in cluster mode. Two process were running at the same time. Because of that, two pools of 100 connections could have been created which is resulting on a total of 200 connections which is higher than the MySQL default connection limit.
Great that found a solution and here's another one with less code.
create a js file, dbconnection.js for example
var mysql = require("mysql");
var pool = mysql.createPool({
connectionLimit: 10,
host: '...',
user: '...',
password: '...',
database: '...',
dateStrings: true
});
exports.connection = {
query: function () {
var queryArgs = Array.prototype.slice.call(arguments),
events = [],
eventNameIndex = {};
pool.getConnection(function (err, conn) {
if (err) {
if (eventNameIndex.error) {
eventNameIndex.error();
}
}
if (conn) {
var q = conn.query.apply(conn, queryArgs);
q.on('end', function () {
conn.release();
});
events.forEach(function (args) {
q.on.apply(q, args);
});
}
});
return {
on: function (eventName, callback) {
events.push(Array.prototype.slice.call(arguments));
eventNameIndex[eventName] = callback;
return this;
}
};
}
};
In the other file where you want to use the connection
var db = require('./dbconnection.js');
And instead of
connection.query
Use
db.connection.query