I've got a website I've written that gets data from a Restful web service. That Restful web service is run off node.js and MySQL using node-mysql. The problem I am running into is with connections running out. For example, my default page does a bunch of lookups to get data. On the 9th refresh of that page, one of my lookups to the Restful API throws this error "No connections available."
Here is kind of how things are architected on the Restful API side:
In server.js:
var db = require('mysql');
var db_pool = db.createPool(
{
host : 'localhost',
connectionLimit: 100,
supportBigNumbers: true,
waitForConnections: false
});
setRoutes(server, db_pool, passport, LocalStrategy, jwt, tokenSecret, bcrypt);
In router.js:
setRoutes = function(server, connectionPool, passport, LocalStrategy, jwt, tokenSecret, bcrypt)
{
require('./lib/places.js');
place.connectionPool = connectionPool;
}
server.get('api/place/:id', function(request, response)
{
response.header("Access-Control-Allow-Origin", "*");
response.header("Access-Control-Allow-Headers", "X-Requested-With");
place.get(request.params.id, function(err, data)
{
response.send(data);
next();
});
});
Finally, in places.js:
place.connectionPool = null;
place.get = function(id, callback)
{
place.connectionPool.getConnection(function(error, connection)
{
var query = "select myColumn from myTable";
connection.query
(
query,
function (queryError, rows, fields)
{
try {connection.release();} catch(e){};
if (queryError)
{
console.log(JSON.stringify(queryError));
return (callback(null, queryError));
}
return (callback(null, rows));
}
);
});
}
What is the best practice for implementing node-mysql and connection pooling for websites?
Thanks!
Solved the connection issue - I was not releasing the connection in all places where I was doing a callback.
Related
I have implemented load balancing in read database connection like when read db load increased to 60% it will initiate a new read database for balancing load on database but
When I see from AWS developer console dashboard all API calls It will initate new read database instance but most of the API's calls load took placed on database 1 upto 90 percent but like 10 req /sec and on read DB instance 2 1 to 5% database is used like 1req /sec
it should divided API request on both database equaly but It wont work
This issue is because mysql.createPool will not close connection from database 1 (createPool will reuse its opened connections) so that other API calls can move to second database instance.
To solve this problem I had changed mysql.createPool with mysql.createConnection on Each API calls
I had created 2 middleware
1-for createConnection
2-for connection.end()
whenever a request comes in middleware 1 calls and create new connection and on request finish middleware 2 will call which will end the connection. this solution has solved my problem of load balancing but a new issue takes place I have face to many database connection issues with this method
does anyone have a proper solution who has faced this issue or can help?
Sample Code :
var readDB = mysql.createConnection({
database: process.env.READ_DB_DATABASE,
host: process.env.READ_DB_HOST,
user: process.env.READ_DB_DB_USER,
password: process.env.READ_DB_DB_PASSWORD,
charset: "utf8mb4"
});
utils.js
async onFinish(req, res, next) {
return new Promise(async (resolve, reject) => {
try {
let readDB = req.readDB;
const dbEnd = util.promisify(readDB.end).bind(readDB);
const response = await dbEnd();
resolve(response);
} catch (error) {
reject(error);
}
});
}
app.js
/**
* middleware for create connection and end connection on finish
*/
app.use(async (req, res, next) => {
try {
const readDB = await utils.readDBCreateConnection();
req.readDB = readDB;
res.on("finish", function () {
console.log("onFinish called");
utils.onFinish(req, res, next);
});
next();
} catch (error) {
res.status(error.status || 500).send({
code: 500,
message: error.message || `Internal Server Error`,
});
}
});
/**
* Code to initialice routing
*/
require("./modules/v2-routes")(app); // v2 app routes
My web app uses Lambda using NodeJS and backend is RDS(MySQL). I'm using serverless-mysql to make db calls.
For some reason, the db call times out intermittently. I tried the following:
Enabled flow logs to see if there are any errors (but couldn't find any reject statuses).
Tried making the database publicly available and took lambda out of VPC (to see if it is an issue with VPC configuration). But still, it was failing intermittently. So VPC is out of the equation.
RDS is not having any unusual spikes and connection exhaustion as monitoring shows a peak of only up to 3 connections. Lambda is always kept warm. I tried increasing the time out to up to 25 seconds. Still no luck.
Below is the code I use:
export async function get(event, context, callback) {
if (await warmer(event)) return 'warmed';
context.callbackWaitsForEmptyEventLoop = false;
try {
const userId = getUserIdFromIdentityId(event);
const query = "select * from UserProfile where UserId = ?";
const result = await mysql.query(query, [userId]);
console.log(result);
console.log('getting user account');
mysql.quit();
return success({
profileSettings: result.length > 0 ? result[0] : null,
});
} catch(e) {
console.log(e);
return failure();
}
}
Success function basically returns a json object like below:
return {
statusCode: 200,
headers: {
"Access-Control-Allow-Origin": "*",
"Access-Control-Allow-Credentials": true
},
body: JSON.stringify(body)
};
mysql is initialized as below:
export const mysql = AWSXray.captureMySQL(require('serverless-mysql')({
config: {
host: process.env.dbHost,
user: process.env.dbUsername,
password: process.env.dbPassword,
database: process.env.database,
}
}));
The only error I can see in Cloudwatch logs is:
Task timed out after 10.01 seconds.
Im a android dev and trying to make a simple rest api with node js, so Im basically new to js.
Im setting up a new rest api and want to connect to mysql database.
I was trying to solve that this way, but I'm getting errors.
And, also how many connection limits to set ?
const express = require('express');
const db = require('../db');
const mainNewsRouter = express.Router();
mainNewsRouter.get('/', async (req, res, next) => {
try {
let result = await db.getMainNews();
console.log(res.json(result));
res.json(result);
} catch(e) {
console.log(e);
}
});
module.exports = mainNewsRouter;
//DbHandler.js
var mysql = require('mysql2');
const url = require('url');
var SocksConnection = require('socksjs');
var remote_options = {
host:'xxx',
port: 3306
};
var proxy = url.parse('http://xxx:xxx#us-east-static-06.quotaguard.com:xxx');
var auth = proxy.auth;
var username = auth.split(":")[0];
var pass = auth.split(":")[1];
var sock_options = {
host: proxy.hostname,
port: 1080,
user: username,
pass: pass
};
var sockConn = new SocksConnection(remote_options, sock_options);
var dbConnection = mysql.createPool({
connectionLimit: 10,
user: 'xxx',
database: 'xxx',
password: 'xxx',
stream: sockConn
});
getMainNews = () => {
return new Promise((resolve, reject) => {
dbConnection.query('SELECT ... * from ...;',
(err, results) => {
if (err) {
return reject(err);
};
// sockConn.dispose();
return resolve(results);
});
});
dbConnection.end();
};
On first api call I get data from database, but with this error:
Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client
at ServerResponse.setHeader (_http_outgoing.js:470:11)
at ServerResponse.header (node_modules\express\lib\response.js:771:10)
at ServerResponse.send (node_modules\express\lib\response.js:170:12)
at ServerResponse.json (node_modules\express\lib\response.js:267:15)
at mainNewsRouter.get (server\routes\mainNews.js:10:11)
at process._tickCallback (internal/process/next_tick.js:68:7)
And after second API call there is no data, I only get this exception.
> Server is running on port: { Error: This socket has been ended by the
> other party
> at Socket.writeAfterFIN [as write] (net.js:395:12)
> at SocksConnection._write (node_modules\socksjs\socks.js:72:24)
> at doWrite (_stream_writable.js:415:12)
> at writeOrBuffer (_stream_writable.js:399:5)
> at SocksConnection.Writable.write (_stream_writable.js:299:11)
> at PoolConnection.write (node_modules\mysql2\lib\connection.js:221:17)
> at PoolConnection.writePacket(node_modules\mysql2\lib\connection.js:279:12)
> at ClientHandshake.sendCredentials (node_modules\mysql2\lib\commands\client_handshake.js:63:16)
> at ClientHandshake.handshakeInit (node_modules\mysql2\lib\commands\client_handshake.js:136:12)
> at ClientHandshake.execute (node_modules\mysql2\lib\commands\command.js:39:22) code: 'EPIPE',
> fatal: true }
Although I am by no means an expert, I think one of the issues lies with closing the connection. The whole idea of a pool is to release the connection back to the pool, not close it.
I have done testing on connection pools and have used a pool size of min:4 max:12 with 100s of requests per second without running into connections issues with MySQL.
Personally, I use Knex to manage my db connections, it manages all of the pools too, taking care of a lot of the headache. Low overhead, I think it would be worth porting over that part of your code to. Once the connection issue is sorted out, then you could tackle other issues as they crop up.
Again, I am not an expert and cannot exactly nail down releasing the MySQL connection back to the pool in the code above, but I do think that is why you don't get data after your initial call.
It wont answer your full question but still. The "Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client" means that a header has already been set but the user is again trying to set it. A header is set when we send a response. Headers include the content type, content-length,status and all the information about the response we are sending. When we write res.send or res.json or res.render i.e sending a response the headers get set automatically using the required information (Express does it automatically for us, in pure nodejs we have to set every header by ourselves). Notice that you have written res.json two times which means it has to set the headers twice. Also writing res.json inside console.log doesnt make any sense. Why have you done that?
running into some issues trying to figure out an Azure Function (node.js-based) can connect to our mysql database (also hosted on Azure). We're using mysql2 and following tutorials pretty much exactly (https://learn.microsoft.com/en-us/azure/mysql/connect-nodejs, and similar) Here's the meat of the call:
const mysql = require('mysql2');
const fs = require('fs');
module.exports = async function (context, req) {
context.log('JavaScript HTTP trigger function processed a request.');
if (req.query.fname || (req.body && req.body.fname)) {
context.log('start');
var config = {
host:process.env['mysql_host'],
user: process.env['mysql_user'],
password: process.env['mysql_password'],
port:3306,
database:'database_name',
ssl:{
ca : fs.readFileSync(__dirname + '\\certs\\cacert.pem')
},
connectTimeout:5000
};
const conn = mysql.createConnection(config);
/*context.log(conn);*/
conn.connect(function (err) {
context.log('here');
if (err) {
context.error('error connecting: ' + err.stack);
context.log("shit is broke");
throw err;
}
console.log("Connection established.");
});
context.log('mid');
conn.query('SELECT 1+1',function(error,results,fields) {
context.log('here');
context.log(error);
context.log(results);
context.log(fields);
});
Basically, running into an issue where the conn.connect(function(err)... doesn't return anything - no error message, no logs, etc. conn.query works similarly.
Everything seems set up properly, but I don't even know where to look next to resolve the issue. Has anyone come across this before or have advice on how to handle?
Thanks!!
Ben
I believe the link that Baskar shared covers debugging your function locally
As for your function, you can make some changes to improve performance.
Create the connection to the DB outside the function code otherwise it will create a new instance and connect every time. Also, you can enable pooling to reuse connections and not cross the 300 limit that the sandbox in which Azure Functions run has.
Use the Promises along with async/await
You basically can update your code to something like this
const mysql = require('mysql2/promise');
const fs = require('fs');
var config = {
host: process.env['mysql_host'],
user: process.env['mysql_user'],
password: process.env['mysql_password'],
port: 3306,
database: 'database_name',
ssl: {
ca: fs.readFileSync(__dirname + '\\certs\\cacert.pem')
},
connectTimeout: 5000,
connectionLimit: 250,
queueLimit: 0
};
const pool = mysql.createPool(config);
module.exports = async function(context, req) {
context.log('JavaScript HTTP trigger function processed a request.');
if (req.query.fname || (req.body && req.body.fname)) {
context.log('start');
const conn = await pool.getConnection();
context.log('mid');
await conn.query('SELECT 1+1', function(error, results, fields) {
context.log('here');
context.log(error);
context.log(results);
context.log(fields);
});
conn.release();
}
};
PS: I haven't test this code as such but I believe something like this should work
Debugging on serverless is challenging for obvious reasons. You can try one of the hacky solutions to debug locally (like Serverless Framework), but that won't necessarily help you if your issue is to do with a connection to a DB. You might see different behaviour locally.
Another option is to see if you can step debug using Rookout, which should let you catch the full stack at different points in the code execution and give you a good sense of what's failing and why.
I am very new to Node.js development and I am working on an app that requires me to pull users from a mysql database. I am using the promise-mysql library to query a mysql database. I am trying to use a connection pool like this:
var pool = mysql.createPool({
host: hgh.host,
user: hgh.user,
password: hgh.pw,
database: hgh.name,
connectionLimit: 10
});
As a global variable in my module.
I then have the above function to return a connection from the pool.
function connect() {
return pool.getConnection().then(function(connection) {
return connection
}).catch(function(error) {
console.log("Connect failed");
throw ErrorModel.generateErrorObject(error, 500);
});
}
Below is a function I am using to query the database with:
function getUser(username) {
var sql_query = `SELECT * FROM userstable WHERE userName = ` + `'` + username + `'`;
return connect().then(function(conn) {
return conn.query(sql_query).then(function(rows) {
console.log("HGH getUser Then");
console.log(pool);
conn.release();
return rows;
});
}).catch(function(error) {
console.log("HGH getUser Catch");
console.log(error);
throw ErrorModel.generateErrorObject(error, 500);
});
}
I am getting this issue:
conn.release is not a function when trying to release my current connection into the pool. Is my logic wrong here? My goal is to have a pool with a bunch of connections (up to a certain number) and if a user needs to query, the getConnection() function just grabs them either a free connection from the pool, or creates them one. Problem is I cannot for the life of me release it back to the pool as free..Every time I make a request with conn.end() instead of release the connection remains in the _allConnections array of the pool when I console it out, and there are absolutely no connections in the _freeConnections array.
Anyone know how I can make connections free again??
Looking at the module's code I found the function for releasing a connection from a pool:
pool.prototype.releaseConnection = function releaseConnection(connection) {
//Use the underlying connection from the mysql-module here:
return this.pool.releaseConnection(connection.connection);
};
So if all of these functions live in the same file you could do the following in the getUser function:
replace
conn.release();
with
pool.releaseConnection(conn);
Looking at the code, promise-mysql wraps the original connection object, which doesn't expose the release method. However, the original is exposed as a property called connection, so this works:
conn.connection.release();
A few random thoughts:
You should probably escape your query input:
var sql_query = `SELECT * FROM userstable WHERE userName = ${ pool.escape(username) }`;
Your code doesn't release connections when an error occurs (because the code in the .then() callback wouldn't get called); it's better to use .finally() to do the releasing, as that will get called for both resolved and rejected cases:
function connect() {
var conn = null;
return pool.getConnection().then(function(connection) {
conn = connection;
return connection;
}).catch(function(error) {
console.log("Connect failed", error);
}).finally(function() {
if (conn) {
conn.connection.release();
}
});
}