I am trying to learn how to connect MySQL using lambda functions in AWS. I have followed a couple of instructions online and basically ended up with this code:
var mysql = require('mysql');
var pool = mysql.createPool({
connectionLimit : 1000,
connectTimeout : 60 * 60 * 1000,
acquireTimeout : 60 * 60 * 1000,
timeout : 60 * 60 * 1000,
host: "foo-bar-123.us-east-2.rds.amazonaws.com",
user: "root",
password: "pass123",
database: "sample_db",
});
exports.handler = (event, context, callback) => {
// prevent timeout from waiting event loop
context.callbackWaitsForEmptyEventLoop = false;
pool.getConnection(function(err, connection) {
if (err) throw err;
// Use the connection
connection.query('SELECT id FROM customer limit 10;', function (error, results, fields) {
// And done with the connection.
connection.release();
// Handle error after the release.
if (error) callback(error);
else callback(null,results);
});
});
};
This is working on my local but when I zip this code and uploaded it as a lambda function, this returns
Response:
{
"errorMessage": "2018-11-13T02:16:10.339Z 0562b432-e6ea-11e8-81ef-bd64aa1af0a4 Task timed out after 30.03 seconds"
}
It times out no matter how many seconds I set it to.
I have pretty much set everything at default since I am new to all of these but I have added AmazonRDSFullAccess to the role of lambda function.
Does anyone have any idea what may be wrong or missing in my setup?
Thanks.
After doing some trial and errors, I was able to make it work and what I was missing is that I did not allow All TCP in the inbound of my RDS Security group. After that, I set it as my lambda function to No VPC, and it was able to query properly.
This link: https://dzone.com/articles/querying-rds-mysql-db-with-nodejs-lambda-function and the stack overflow link posted in there (which is this: AWS Lambda RDS connection timeout) helped me a lot in figuring out what was wrong with my code/setup.
Here is the final code that I ended up using.
const mysql = require('mysql');
const pool = mysql.createPool({
host: "foo-bar-123.us-east-2.rds.amazonaws.com",
user: "root",
password: "pass123",
database: "sample_db"
});
exports.handler = (event, context, callback) => {
//prevent timeout from waiting event loop
context.callbackWaitsForEmptyEventLoop = false;
pool.getConnection((err, connection) => {
if(err) throw err;
// Use the connection
connection.query('SELECT id FROM customer limit 10;', (error, results, fields) => {
// And done with the connection.
connection.release();
// Handle error after the release.
if (error) callback(error);
else callback(null,results);
});
});
};
Thanks!
Related
My question is similar to this post but the solution didnt work for me probably because im using a different type of mysql connection (pool). This is my code:
let config= {
host: '***',
user: 'admin',
password: '***',
port: '3306',
database: '***',
multipleStatements: true
};
const con = mysql.createPool(config);
select();
function select(){
return new Promise((resolve, reject) => {
con.getConnection(function (err, connection) {
if (err) throw err;
else
console.log("Connected!");
let sql = "SELECT * FROM bidPrice WHERE idExchangePlatform = 2;";
connection.query(sql, function (err, results, fields) {
connection.release();
connection.destroy();
if (err) throw err;
console.log(results)
resolve(results);
});
});
});
}
I also important to mention that im running this function using the following command
node --max-old-space-size=31744 index.js # Increase to 31 GB
This is because im working with millions of records from the database query If i run this with regular node command i would be getting Javascript heap out of memory
When i tried integrating the solution i mentioned earlier to my code i just get a "killed" log after a while and then the process stops, should i handle server disconnect in a different way when using mysql.pool?
This happens pretty frequently (once a week for about 30-40 minutes), where all of a sudden my database mentions max connections when I try to connect via heidisql, and any apis calls respond with the following error:
Cannot read property 'release' of undefined
I am calling .release() after every query in mysql. Is there something I am missing, am I suppose to call .end as well? I am using nodejs with mysql.
Here is the way I wrap every query and the pool code:
var mysql = require('mysql');
var mysql_pool = mysql.createPool({
connectionLimit: config.mysql.limit,
host: config.mysql.host,
user: config.mysql.user,
password: config.mysql.pw,
database: config.mysql.db //,
// debug: true
});
var qSelect = "SELECT id FROM Users";
var qValues = [];
var qCall = mysql.format(qSelect, qValues);
mysql_pool.getConnection(function(err_pool, connection) {
if (err_pool) {
connection.release();
console.log(' Error getting mysql_pool connection: ' + err_pool);
throw err_pool;
}
connection.query(qCall, function(err, userFound, fields) {
connection.release();
if (err) {
console.log("get user : " + err);
} else {
//some code here
}
});
Can someone please advise, appreciate it.
You should remove first connection.release() used in if loop
if (err_pool) {
console.log(' Error getting mysql_pool connection: ' + err_pool);
throw err_pool;
}
I have a simple program that connects to an AWS MySQL DB via Lambda.
Every request from the app is basically a single query on the database. So if I have 3 users using the app, it would be impossible for more than 3 queries to be run on the database at the same time.
Yet when I look at my connection count it could be as high as 15. I've been having trouble understanding exactly how this should work.
If the code is written correctly does that mean if there's 3 users at once, I shouldn't have more than 3 connections?
I've been trying to find a basic article to explain this but obviously haven't hence I'm here.
Thank you.
EDITED TO ADD CODE
Below is a cut down version of my code
var mysql = require('mysql');
var pool =[];
exports.handler = (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
pool = mysql.createPool({
host : 'host-details',
user : 'username',
password : 'password',
database : 'db'
});
return myFunc(event, context, callback, data.Username);
}
};
const myFunc = async (event, context, callback) => {
const query = 'SELECT * from db.table';
const data = await dbQueryAsyncWL(context, callback, query);
return callback(null, {data: data});
}
const dbQueryAsyncWL = async (context, callback, query) => {
return new Promise((resolve) => {
context.callbackWaitsForEmptyEventLoop = false;
pool.getConnection(function(error, connection) {
connection.query(query, function (error, results, fields) {
connection.release();
if (error) {callback(error);} else {resolve(results);}
});
});
})
}
pool = mysql.createPool({
host : 'host-details',
user : 'username',
password : 'password',
database : 'db'
});
Check the size / minimum size of the connection pool.
What is the connection pool configuration, like idleTime etc before closing a connection.
What trends are you noticing around the size of connection pool, is it stable or increase/decreasing ?
Looks like my issue was that I was creating the pool inside the exports.handler. Once I moved it outside... well a picture says it all.
https://i.ibb.co/KDvfdhp/Screen-Shot-2020-06-29-at-10-26-49-pm.png
I've created a simple mySQL database that I'm trying to access data from via an AWS Lambda function.
This is a version of the code that runs fine locally:
var mysql = require('mysql');
var config = require('./config.json');
var pool = mysql.createPool({
host : config.dbhost,
user : config.dbuser,
password : config.dbpassword,
database : config.dbname
});
pool.getConnection(function(err, connection) {
// Use the connection
connection.query("SELECT username FROM ClimbingDB.users WHERE email = 'testemail1'", function (error, results, fields) {
// And done with the connection.
connection.release();
// Handle error after the release.
if (error) throw error;
console.log(results);
process.exit();
});
});
This is that code converted to work with AWS Lambda:
var mysql = require('mysql');
var config = require('./config.json');
var pool = mysql.createPool({
host : config.dbhost,
user : config.dbuser,
password : config.dbpassword,
database : config.dbname
});
exports.handler = (event, context, callback) => {
//prevent timeout from waiting event loop
context.callbackWaitsForEmptyEventLoop = false;
pool.getConnection(function(err, connection) {
if (err) return callback(err)
// Use the connection
connection.query("SELECT username FROM ClimbingDB.users WHERE email = 'testemail1'", function (error, results, fields) {
// And done with the connection.
connection.release();
// Handle error after the release.
if (error) return callback(error);
else return callback(null,results);
});
});
};
Which times out with this error message:
{
"errorMessage": "2019-07-19T17:49:04.110Z 2f3e208c-62a6-4e90-b8ec-29398780a2a6 Task timed out after 3.00 seconds"
}
I'm not sure why it doesnt seem to be able to connect. I tried adding the function to a vpc and a security group that has access to RDB's, neither of which do anything. I'm not sure what I'm doing wrong here.
You will need:
The Amazon RDS instance in the same VPC as the AWS Lambda function
A security group on the Lambda function (Lambda-SG)
A security group on the RDS instance (DB-SG) that permits inbound connections on port 3306 from Lambda-SG
That is, DB-SG should specifically reference Lambda-SG (it will turn into a security group ID in the format sg-1234).
You might also want to increase the timeout of the Lambda function to give it a bit more time to run.
I deployed a NodeJS API on Heroku and tried to connect it to a MySQL DB, so I created a connectionPool for handling connections on a ClearDB ignite account (free), which allows a maximum of 10 connections.
Every time I execute a query to the database it just adds a new connection to the stack until it reaches 10 connections and the app crashes.
My code is as follows:
connectionFactory:
var mysql = require('mysql');
function createDBConnection() {
var conn = mysql.createPool({
host: 'xxx',
user: 'xxx',
password: 'xxx',
database: 'xxx'
});
return conn;
}
module.exports = function() {
return createDBConnection;
}
And here's my select query:
function Dao(connection) {
this._connection = connection;
}
Dao.prototype.findAll = function (callback) {
this._connection.query('SELECT * FROM table',
function(errors, results) {
callback(errors,results);
});
};
module.exports = function() {
return Dao;
}
Finally here's the route I use to call it:
app.get('/products', function (req,res) {
var connection = app.persistence.connectionFactory();
var dao = new app.persistence.Dao(connection);
dao.findAll(function (err, result) {
res.format({
json: function () {
res.json(result);
}
});
});
});
I tried changing createPool() to createConnection() and then calling .end()/.destroy() function right after each query but it doesn't work at all.
Any hints?
In order to close a connection / return a connection to the pool use : connection.release()
var mysql = require('mysql');
var pool = mysql.createPool(...);
pool.getConnection(function(err, connection) {
// Use the connection
connection.query('SELECT something FROM sometable', function (error, results, fields) {
// And done with the connection.
connection.release();
// Handle error after the release.
if (error) throw error;
// Don't use the connection here, it has been returned to the pool.
});
});
mysql : Pooling connections Documentation