Lambda Node.js Mysql RDS Timeout - mysql

My lambda function written in Node.js times out when connecting to RDS.
Weird thing is, this timeout happens only for the first request.
All subsequent requests work with the DB without the timeout.
Any idea why?
Just FYI not using any VPCs.
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'ahost',
user : 'auser',
password : 'apassword',
database : 'adb',
port : 3306
});
exports.handler = async (event, context) => {
let request = JSON.parse(event.body);
let question = request.question;
let answered = question.answered;
let sId = request.sid;
let questionnaireId = request.questionnaireId;
let hutk = request.hutk;
let questionId = question.question.id;
pool.getConnection((error, connection) => {
if (error) throw error;
let values = [];
if(Array.isArray(answered)){
let i = 0;
while(i < answered.length){
let td = [
questionnaireId,
sId,
questionId,
answered[i],
hutk
];
values.push(td);
i++;
}
} else {
let td = [
questionnaireId,
sId,
questionId,
answered,
hutk
];
values.push(td);
}
let delsql = "DELETE FROM answers WHERE sId= ? AND `key` = ?";
connection.query(delsql, [sId, questionId], function(err, result){
if(err) throw err;
});
let sql = "INSERT INTO answers (qId, sId, `key`, value, hutk) VALUES ?";
connection.query(sql, [values], function(err, result){
if(err) throw err;
console.log("Successfull Insert")
connection.release();
});
});
// TODO implement
const response = {
statusCode: 200,
headers: {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Credentials': true
},
body: JSON.stringify({message : 'success'}),
};
return response;
};

You might be running into and issue where you're releasing your pool connection while (or prior to) one of your two queries is being run.
You don't need to explicitly call getConnection once you've created the pool. More importantly if you have code where queries might execute in parallel (as you do here) you have to be very judicious in managing the connection and only releasing it once you're certain all the queries that will use it have completed.
Read more here: (https://github.com/mysqljs/mysql#pooling-connections)
Consider trying the following:
var mysql = require('mysql');
var pool = mysql.createPool({
host: 'ahost',
user: 'auser',
password: 'apassword',
database: 'adb',
port: 3306
});
pool.on('connection', function (connection) {
console.log('Pool id %d connected', connection.threadId);
});
pool.on('enqueue', function () {
console.log('Waiting for available connection slot');
});
exports.handler = async (event, context) => {
let request = JSON.parse(event.body);
let question = request.question;
let answered = question.answered;
let sId = request.sid;
let questionnaireId = request.questionnaireId;
let hutk = request.hutk;
let questionId = question.question.id;
let values = [];
if (Array.isArray(answered)) {
let i = 0;
while (i < answered.length) {
let td = [
questionnaireId,
sId,
questionId,
answered[i],
hutk
];
values.push(td);
i++;
}
}
else {
let td = [
questionnaireId,
sId,
questionId,
answered,
hutk
];
values.push(td);
}
let delete_query = "DELETE FROM answers WHERE sId= ? AND `key` = ?";
pool.query(delete_query, [sId, questionId], function(err, result) {
if (err) throw err;
});
let insert_query = "INSERT INTO answers (qId, sId, `key`, value, hutk) VALUES ?";
pool.query(insert_query, [values], function(err, result) {
if (err) throw err;
console.log("Successfull Insert")
});
// TODO implement
const response = {
statusCode: 200,
headers: {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Credentials': true
},
body: JSON.stringify({
message: 'success'
}),
};
return response;
};

Related

Foreach loop in nodejs

My Project uses Node JS + My SQL
I am running a simple query that checks whether entries exist in database for some params.
If they exist, it then updates them, else it inserts them.
The problem is, it is entering the last entry n times instead of each being unique.
When updating also, it is not able to identify the right row.
router.post('/feeRegister', asyncMiddleware( async(req, res) => {
let post= {
...........
}
JSON.parse(req.body.fees).forEach((i,index) => {
let sql = `SELECT id, period, amount FROM feeregister WHERE schoolId = '${req.body.schoolId}' AND studentId = '${req.body.student}' AND classes = '${req.body.classes}' AND year = '${req.body.year}' AND type = '${JSON.parse(req.body.fees)[index][1]}';`
pool.query(sql, async(err, results) => {
try{
if(err){ throw err }
if(results){
if(results.length){
console.log('Entry Exists', index, results[0].id)
let post2= {
...............
}
let sql2 = `UPDATE feeregister SET ? WHERE id = '${results[0].id}'`;
pool.query(sql2, post2, async(err, results) => {
try{
if(err){ throw err }
if(results){}
}catch(e){ func.logError(e); res.status(500); return; }
})
}else{
console.log('Entry does not exist', index)
let sql = `INSERT INTO feeregister SET ?`
pool.query(sql, post, async(err, results) => {
try{
if(err){ throw err }
if(results){}
}catch(e){ func.logError(e); res.status(500); return; }
})
}
}
}catch(e){ func.logError(e); res.status(500); return; }
})
});
}))
Iam connecting to database like :
var mysql = require('mysql')
const pool = mysql.createPool({
host: 'localhost',
user: 'root',
database: 'dBName',
password: '',
multipleStatements: true
});
module.exports = pool;
Perhaps you should call the query function with async-await method.
router.post('/feeRegister', asyncMiddleware( async(req, res) => {
let post= {
//
}
JSON.parse(req.body.fees).forEach(async (i,index) => {
let sql = `SELECT id, period, amount FROM feeregister WHERE schoolId = '${req.body.schoolId}' AND studentId = '${req.body.student}' AND classes = '${req.body.classes}' AND year = '${req.body.year}' AND type = '${JSON.parse(req.body.fees)[index][1]}';`
const promisePool = pool.promise();
let results = await promisePool.query(sql);
if(results){
if(results.length){
console.log('Entry Exists', index, results[0].id)
let post2= {
//
}
let sql2 = `UPDATE feeregister SET ? WHERE id = '${results[0].id}'`;
await promisePool.query(sql2);
}else{
console.log('Entry does not exist', index)
let sql = `INSERT INTO feeregister SET ?`
await promisePool.query(sql);
}
}
});}));

convert a NodeJS lambda function (AWS) to use "async" (promises, etc.) instead of callbacks

I have a lambda function that connects to mysql and runs a set of queries, but I actually have a sequence of mysql queries that need to run one after another. I.e., the value of one query is used in the next query, etc.
Currently, I have a bunch of callbacks to achieve this, but this is leading to "callback hell". How would I rewrite this to use async / await?
My code is actually split into 2 files. The first file does an initial query, and then the value is passed into a function of the second file. Please note that the mysql node_module is included but not shown here. The AWS API gateway calls index.js
// index.js
var mysql = require('mysql'); // from node_modules
var config = require('./config.json');
var dashboard = require('./dashboard.js');
var pool = mysql.createPool({
host : config.dbhost,
user : config.dbuser,
password : config.dbpassword,
database : config.dbname
});
exports.handler = (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
pool.getConnection(function(err, connection) {
// check for mysql connection error first
if ( err ) {
throw err;
}
let qry = "select id from some_table where some_field = ?";
let someval = event.queryStringParameters.someval;
connection.query(qry, [someval], function(error, result) {
if ( error ) {
throw err;
}
else {
dashboard.processRequest(connection, callback, event, res[0].id);
}
});
});
}
// dashboard.js
module.exports = {
jsonResponse: function(results) {
return {
"statusCode": 200,
"body": JSON.stringify({ results }),
"isBase64Encoded": false,
"headers": {
"Access-Control-Allow-Origin": "*"
}
};
},
processRequest: function(connection, callback, event, val) {
let qry = "update first_table set some_field = ?";
connection.query(qry, [val], function(error, results) {
// return to client if error
if (error) {
callback(null, this.jsonResponse(error));
}
else {
// assume that this table must be update AFTER the previous statement
qry = "select id from second_table where some_field = ?";
connection.query(qry, [val], function(error1, results1) {
// return to client if error
if ( error1 ) {
callback(null, this.jsonResponse(error1));
}
qry = "update third_table set some_field = ? where id = ?";
connection.query(qry, [results1[0].id], function(error2, results2) {
// release connection when all queries are completed
connection.release();
if ( error2 ) {
callback(null, this.jsonResponse(error2));
}
else {
callback(null, this.jsonResponse(results2));
}
});
});
}
});
}
};
It was suggested to me that something like the below code might work. Unfortunately, it does not. I was curious to know why using try...catch blocks in the way shown below is not working, and is it the same thing as what you've shown, but just written differently?
// index.js
var mysql = require('mysql'); // from node_modules
var config = require('./config.json');
var dashboard = require('./dashboard.js');
var pool = mysql.createPool({
host : config.dbhost,
user : config.dbuser,
password : config.dbpassword,
database : config.dbname
});
exports.handler = (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
pool.getConnection(function(err, connection) {
// check for mysql connection error first
if ( err ) {
throw err;
}
let qry = "select id from users where username = ? limit 1;";
let username = event.queryStringParameters.username;
try {
let res = await connection.query(qry, [event.queryStringParameters.username]);
dashboard.processRequest(connection, callback, event, res[0].id);
} catch (err) {
console.log(err);
}
});
}
// dashboard.js
module.exports = {
jsonResponse: function (results) {
return {
"statusCode": 200,
"body": JSON.stringify({results}),
"isBase64Encoded": false,
"headers": {
"Access-Control-Allow-Origin": "*"
}
};
},
processRequest: async function (connection, callback, event, val) {
let qry = "update first_table set some_field = ?";
try {
let results = await connection.query(qry, [val]);
qry = "select id from second_table where some_field = ?";
try {
let results1 = await connection.query(qry, [val]);
qry = "update third_table set some_field = ? where id = ?";
try {
let results2 = await connection.query(qry, [results1[0].id]);
connection.release();
callback(null, this.jsonResponse(results2));
} catch (error2) {
callback(null, this.jsonResponse(error2));
}
} catch (error1) {
callback(null, this.jsonResponse(error1));
}
} catch (error) {
callback(null, this.jsonResponse(error));
}
}
};
We need use promises.
Typically I follow this approach:
Create one async method mainProcess and have bunch of methods step by step called with in that method. one after the other with await or all at once.
Each async method getConnection and runQuery in this case, called within mainProcess must a Promise.
If any errors from these methods i.e promise rejects from individual methods, goes in catch block of mainProcess().
If no errors, all methods within mainProcess gets executed and goes to then block of mainProcess()
Your code can be refactored like this (just wrote in an editor untested)
var pool = mysql.createPool({
host: config.dbhost,
user: config.dbuser,
password: config.dbpassword,
database: config.dbname,
});
exports.handler = (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
/**
* Main Lambda Process
*/
const mainProcess = async () => {
// Get Connection
let connection = await getConnection();
// Run Step 1
let qry1 = "select id from some_table1 where some_field = ?";
const response1 = await runQuery(connection, qry1, { someFiledValue: 1222})
// Run Step 2
let qry2 = "select id from some_table2 where some_field = ?";
const resonse2 = await runQuery(connection, qry2, { someFiledValue: 1222})
return 'All Good';
});
}
mainProcess()
.then(result => {
// All lambda success messages are returned from here
callback(null, result);
})
.catch(error => {
// All lambda errors thrown from here
callback(error);
});
};
function getConnection(qry, parms) {
return new Promise((resolve, reject) => {
pool.getConnection(function (error, connection) {
if (error) {
// return to client if error
reject(error);
} else {
// Return response
resolve(connection);
}
});
});
}
/**
* Runs a query, either resolves or rejects
*/
function runQuery(connection, qry, parms) {
return new Promise((resolve, reject) => {
connection.query(qry, [val], function (error, results) {
if (error) {
// return to client if error
reject(error);
} else {
// Return response
resolve(result);
}
});
});
}
When you're dealing with a lambda function which performs an async task you have two solutions:
you can use non async handlers, in which case you need to invoke "callback" on promises as you did in your example
you can use async handlers, which does not requires the "callback" input and that allows you to write async/await code, like the following example:
const mysql = require('mysql2/promise');
exports.handler = async(event, context) => {
//get path variable
const { pathVar } = event.pathParameters;
// get connection
const connection = await mysql.createConnection({
host : process.env.RDS_HOSTNAME,
user : process.env.RDS_USERNAME,
password : process.env.RDS_PASSWORD,
database : process.env.RDS_DB_NAME
});
// get text query
const textQuery = `SELECT field FROM entity WHERE attribute = ${pathVar}`;
// get res
const results = await connection.execute(textQuery);
return {
"statusCode": 200,
"body": results,
"isBase64Encoded": false
}
}
You can have a look at the AWS docs: https://docs.aws.amazon.com/lambda/latest/dg/nodejs-handler.html

Nodejs 12 Callback not working for mysql connection?

I'm writing this code to run in AWS Lambda.
Looking at mysql connection documentation
I expect, if things are working with out errors, to get the message "Database connected successfully!"
and then to get a message "connected as id " but that is not the order that it is happening. Here is my code.
'use strict';
let mysql = require('mysql');
const connection = mysql.createConnection({
dateStrings: true,
host : process.env.rds_host,
user : process.env.rds_user,
password : process.env.rds_password,
database : process.env.rds_database,
port : process.env.rds_port
});
exports.handler = (event, context, callback) => {
//prevent timeout from waiting event loop
context.callbackWaitsForEmptyEventLoop = false;
let sql = 'SELECT * FROM company ORDER BY points DESC, name ASC';
let data = null;
console.log('\nGetCompanies SQL: ', sql);
let responseBody = "";
let statusCode = 0;
connection.connect(function(err) {
if (err) {
statusCode = 500;
responseBody = err;
}
else{
console.log("Database connected successfully!");
statusCode = 200;
responseBody = "Database connected successfully!";
}
});
console.log('connected as id ' + connection.threadId);
connection.query(sql, data, function(queryError, results) {
if(queryError) {
console.error(queryError.message);
callback(queryError);
}
else
{
console.log('\nGetCompanies Results: ', results[0]);
callback(null, results);
}
});
};
Here is the logged output:
INFO
GetCompanies SQL: SELECT * FROM company ORDER BY points DESC, name ASC 2020-01-01T11:52:57.813Z
INFO connected as id null 2020-01-01T11:52:57.952Z
INFO Database connected successfully! 2020-01-01T11:52:57.974Z
My thought was that the function that I supply to:
connection.connect(function(err) {
would execute before any code after connection.connect. Am I wrong to think that?
One more question: Why is the
connected as id null? I got that code (connection.threadId) straight from the mysql docs.
I know this is no big deal if all it effects is the order of log messages but I have other functions where I can't make queries because the connection isn't there yet.
It's quite confusing when it blows right past the
if (err) {
statusCode = 500;
responseBody = err;
}
else{
<do some connection.query code here>
}
I put these log messages here because this method works and my other method doesn't.
You are getting that behaviour because console.log('connected as id ' + connection.threadId); is not waiting for connection.connect to finish connecting.
You need to use promises here.
You could try to use async/await.
exports.handler = async (event, context, callback) => {
//prevent timeout from waiting event loop
context.callbackWaitsForEmptyEventLoop = false;
let sql = 'SELECT * FROM company ORDER BY points DESC, name ASC';
let data = null;
console.log('\nGetCompanies SQL: ', sql);
let responseBody = "";
let statusCode = 0;
try {
await connection.connect();
console.log("Database connected successfully!");
console.log('connected as id ' + connection.threadId);
const queryResult = await connection.query(sql, data);
// do something with queryResult
} catch (e) {
// handle error
}
};
EDIT
The resource you linked to suggests to use promises over callbacks:
await any promise instead of using callbacks
would execute before any code after connection.connect. Am I wrong to think that?
The function you are passing to the connection.connect is called a
callback function. what that mean is, it will be called only after a
successful connection(or after the connection attempt is errored) is made. So the answer is no. The connection.connect is called and then immediately it will call the next statement. It will not wait for the current statement, because javascript is event driven and non blocking.
connected as id null? I got that code (connection.threadId) straight from the mysql docs.
This is because of the previous statement, you are logging the connectionId before a connection is obtained.
using Callbacks
Your code should be like the below. I haven't modified much, the connection to the database is made only after the callback of the connect event is called.
exports.handler = (event, context, callback) => {
//prevent timeout from waiting event loop
context.callbackWaitsForEmptyEventLoop = false;
let sql = 'SELECT * FROM company ORDER BY points DESC, name ASC';
let data = null;
console.log('\nGetCompanies SQL: ', sql);
let responseBody = "";
let statusCode = 0;
connection.connect(function (err) {
// we are inside the callback function a successful connection has been obtained , or error connecting to database
if (err) {
statusCode = 500;
responseBody = err;
}
else {
console.log("Database connected successfully!");
statusCode = 200;
responseBody = "Database connected successfully!";
}
// connection exists
console.log('connected as id ' + connection.threadId);
connection.query(sql, data, function (queryError, results) {
if (queryError) {
console.error(queryError.message);
callback(queryError);
}
else {
console.log('\nGetCompanies Results: ', results[0]);
callback(null, results);
}
});
});
};
using promises
If you could use promises to make your code readable and understandable.
import { promisify } from 'util';
exports.handler = async (event, context) => {
//prevent timeout from waiting event loop
context.callbackWaitsForEmptyEventLoop = false;
let sql = 'SELECT * FROM company ORDER BY points DESC, name ASC';
let data = null;
console.log('\nGetCompanies SQL: ', sql);
const connect = promisify(connection.connect);
const query = promisify(connection.query);
try {
const connection = await connect();
// connection exists
console.log('connected as id ' + connection.threadId);
const results= await query(sql);
return {
statusCode: 200,
responseBody: JSON.stringify(results)
}
} catch (err) {
return {
statusCode: 500,
responseBody: err.message
}
}
};
You should move your connection.query statement into the connection.connect scope as even if you can not connect to database you will try to make a query which is not working properly.
exports.handler = (event, context, callback) => {
//prevent timeout from waiting event loop
context.callbackWaitsForEmptyEventLoop = false;
let sql = 'SELECT * FROM company ORDER BY points DESC, name ASC';
let data = null;
console.log('\nGetCompanies SQL: ', sql);
let responseBody = "";
let statusCode = 0;
connection.connect(function(err) {
if (err) {
statusCode = 500;
responseBody = err;
}
else{
console.log("Database connected successfully!");
statusCode = 200;
responseBody = "Database connected successfully!";
connection.query(sql, data, function(queryError, results) {
if(queryError) {
console.error(queryError.message);
callback(queryError);
}
else
{
console.log('\nGetCompanies Results: ', results[0]);
callback(null, results);
}
});
}
});
console.log('connected as id ' + connection.threadId);
};

Nodejs mysql how to handle timeout and Handshake inactivity timeout

I am a newbie in Nodejs and I have a lambda function written on NodeJS that's supposed to delete some rows and insert some data on a mysql db.
I have come across various instances with error messages such as PROTOCOL_SEQUENCE_TIMEOUT, PROTOCOL_CONNECTION_LOST and an instance where the RDS db dns couldn't be resolved and connect to the db.
I was wondering how I might handle these events so I'd be able to re-connect and proceed.
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'somehost',
user : 'someuser',
password : 'somepassword',
database : 'somedb',
port : 3306
});
pool.on('connection', function (connection) {
console.log('Pool id %d connected', connection.threadId);
});
pool.on('enqueue', function () {
console.log('Waiting for available connection slot');
});
exports.handler = async (event, context) => {
context.callbackWaitsForEmptyEventLoop = false;
let request = JSON.parse(event.body);
/** SOME OTHER LOGIC HERE **/
let delete_query = "DELETE FROM answers WHERE sId= ? AND `key` = ?";
pool.query(delete_query, [sId, questionId], function(err, result){
if(err) throw err;
});
let insert_query = "INSERT INTO answers (qId, sId, `key`, value, hutk) VALUES ?";
pool.query(insert_query, [values], function(err, result){
if(err) throw err;
console.log("Successfull Insert")
});
const response = {
statusCode: 200,
headers: {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Credentials': true
},
body: JSON.stringify({message : 'success'}),
};
return response;
};
And also am I using the best approach to connecting to the db as in a pool or should I be using just a connection etc?
You can cache the connection, so the first call to your lambda would create the connection and the 2nd call (if the lambda is not cold started) can reuse the connection and is much faster.
Here is how we do it:
const mysql = require('mysql');
const util = require('util');
let mySQLconnection = undefined;
exports.handler = async function handler(event, context) {
try {
getMySQLConnection();
const queryResult = await mySQLconnection.query('Select * from yourtable where id = ? and attribute = ?', [id, attribute]);
} catch (error) {
console.log('ERROR: ' + error);
}
};
function getMySQLConnection() {
if (mySQLconnection !== undefined) {
return;
}
mySQLconnection = mysql.createConnection(yourConnectionJson);
mySQLconnection.query = util.promisify(mySQLconnection.query);
}
You could also do a connection retry in the catch block.

Nodejs lambda function timing out

Just trying out Lambda and nodejs.
I have a timeout error that happens every few times.
My nodejs code is below.
var mysql = require('mysql');
var pool = mysql.createPool({
connectionLimit: 100,
host : 'hostname',
user : 'username',
password : 'password',
database : 'database',
port : 3306
});
exports.handler = async (event, context, callback) => {
let request = JSON.parse(event.body);
let sId = request.sid;
let questionnaireId = request.questionnaireId;
pool.getConnection((error, connection) => {
if (error) throw error;
let sql = "INSERT INTO answers (qId, sId,) VALUES(" + questionnaireId + ", " + sid + ")";
connection.query(sql, function(err, result){
if(err) throw err;
console.log("Successfull Insert")
connection.release();
});
});
const response = {
statusCode: 200,
headers: {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Credentials': true
},
body: JSON.stringify({message : 'success'}),
};
return response;
};
Cloudwatch shows the below message.
"errorMessage": "connect ETIMEDOUT",
"code": "ETIMEDOUT",
Thanks in advance.
Try removing callback argument from your handler. You are returning a promise from your function, so you don't need it.
Defining callback as an argument probably makes aws think you are gonna call it when your function finishes executing, and as you don't it produces a timeout error.
Replace
return response;
with this:
callback(null, response)
Reference: https://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-handler.html