I have multiple modules which require my custom PromisifiedMySQL.js module. I am trying to use a connection pool from the NodeJS mysql library.
I tried setting a variable called connectionPool to hold the initialized pool. However it seems that for every module that imports my PromisifiedMySQL.js file it attempts to initialize a new pool instance.
I want ONLY one pool instance to be created which is contained inside PromisifiedMySQL.js
Why is my connectionPool not keeping its state properly?
const mysql = require('mysql');
let connectionPool = null;
const initializePool = ()=>{
if(isLiveEnv()){
connectionPool = createLivePool();
}
else if(isDevEnv()){
connectionPool = createDevPool();
}
}
if(connectionPool === null){
console.log(connectionPool === null);
console.log('init pool!');
initializePool();
}
let query = (sql, args) =>{
return new Promise((resolve,reject) => {
getConnection().then(con => {
con.query(sql,args,(err,result) => {
if(err) {
con.release();
reject(err);
}
else{
con.release();
resolve(result);
}
})
})
.catch(err =>{
reject(err);
})
})
} // end query.
My export look like this:
module.exports = {
query:query
}
Where query gets a connection from the pool, runs the query and releases the connection.
If I have module1.js and module2.js require('./Promisified.js') then
it actually makes two pools even though after the first one is created I set
connectPool is be non-null;
Related
I'd like to know how these approaches work, as reading the documentation just confuses me exactly what happens.
1. Pool module, required where needed
Does a pool get created each time we require the pool? If the pool module is required in 3 places, will 3 separate pools exist?
pool.js
const pool = mysql.createPool(db);
pool.query = util.promisify(pool.query);
module.exports = pool;
api.js
const pool = require("../pool");
await pool.query(query);
2. Route middleware, with a passed in request property
This way there is only one pool (I think), and only that one is used. Does it work as I think it does?
app.js
const middleware = require("./middleware.js");
let app = express();
let pool = mysql.createPool(db);
app.all("/api/*", middleware(pool));
middleware.js
module.exports = (pool) => {
return (req, res, next) => {
pool.getConnection((err, conn) => {
const asyncQuery = util.promisify(conn.query).bind(conn);
req.asyncQuery = asyncQuery; // <--------- pass pool up the chain
req.connection = conn;
next();
});
};
};
api.js
router.get("/api/route", async (req, res, next) => {
try {
await req.asyncQuery(query); // <-------- use the passed in pool
res.status(200).send(rows);
} catch (err) {
next(err);
} finally {
req.connection.release();
}
});
Our stack is nodejs with MySQL we're using MySQL connections pooling our MySQL database is managed on AWS aurora .
in case of auto failover the master DB is changed the hostname stays the same but the connections inside the pool stays connected to the wrong DB.
The only why we found in order to reset the connection is to roll our servers.
this is a demonstration of a solution I think could solve this issue
but I prefer a solution without the set interval
const mysql = require('mysql');
class MysqlAdapter {
constructor() {
this.connectionType = 'MASTER';
this.waitingForAutoFaileOverSwitch = false;
this.poolCluster = mysql.createPoolCluster();
this.poolCluster.add(this.connectionType, {
host: 'localhost',
user: 'root',
password: 'root',
database: 'app'
});
this.intervalID = setInterval(() => {
if(this.waitingForAutoFaileOverSwitch) return;
this.excute('SHOW VARIABLES LIKE \'read_only\';').then(res => {
// if MASTER is set to read only is on then its mean a fail over is accoure and swe need to switch all connection in poll to secondry database
if (res[0].Value === 'ON') {
this.waitingForAutoFaileOverSwitch = true
this.poolCluster.end(() => {
this. waitingForAutoFaileOverSwitch = false
});
};
});
}, 5000);
}
async excute(query) {
// delay all incoming request until pool kill all connection to read only database
if (this.waitingForAutoFaileOverSwitch) {
return new Promise((resolve, reject) => {
setTimeout(() => {
this.excute(query).then(res => {
resolve(res);
});
}, 1000);
});
}
return new Promise((resolve, reject) => {
this.poolCluster.getConnection(this.connectionType, (err, connection) => {
if (err) {
reject(err);
}
connection.query(query, (err, rows) => {
connection.release();
if (err) {
reject(err);
}
resolve(rows);
});
});
});
}
}
const adapter = new MysqlAdapter();
Is there any other programmable way to reset the connection inside the pool?
Is there any notification we can listing to In case of auto-failover?
Instead of manually monitoring the DB health, as you have also hinted, ideally we subscribe to failover events published by AWS RDS Aurora.
There are multiple failover events listed here for the DB cluster: Amazon RDS event categories and event messages
You can use and test to see which one of them is the most reliable in your use case for triggering poolCluster.end() though.
I have a simple program that connects to an AWS MySQL DB via Lambda.
Every request from the app is basically a single query on the database. So if I have 3 users using the app, it would be impossible for more than 3 queries to be run on the database at the same time.
Yet when I look at my connection count it could be as high as 15. I've been having trouble understanding exactly how this should work.
If the code is written correctly does that mean if there's 3 users at once, I shouldn't have more than 3 connections?
I've been trying to find a basic article to explain this but obviously haven't hence I'm here.
Thank you.
EDITED TO ADD CODE
Below is a cut down version of my code
var mysql = require('mysql');
var pool =[];
exports.handler = (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
pool = mysql.createPool({
host : 'host-details',
user : 'username',
password : 'password',
database : 'db'
});
return myFunc(event, context, callback, data.Username);
}
};
const myFunc = async (event, context, callback) => {
const query = 'SELECT * from db.table';
const data = await dbQueryAsyncWL(context, callback, query);
return callback(null, {data: data});
}
const dbQueryAsyncWL = async (context, callback, query) => {
return new Promise((resolve) => {
context.callbackWaitsForEmptyEventLoop = false;
pool.getConnection(function(error, connection) {
connection.query(query, function (error, results, fields) {
connection.release();
if (error) {callback(error);} else {resolve(results);}
});
});
})
}
pool = mysql.createPool({
host : 'host-details',
user : 'username',
password : 'password',
database : 'db'
});
Check the size / minimum size of the connection pool.
What is the connection pool configuration, like idleTime etc before closing a connection.
What trends are you noticing around the size of connection pool, is it stable or increase/decreasing ?
Looks like my issue was that I was creating the pool inside the exports.handler. Once I moved it outside... well a picture says it all.
https://i.ibb.co/KDvfdhp/Screen-Shot-2020-06-29-at-10-26-49-pm.png
I'm fairly new to how database connections work using nodejs, and I'm having issues with database connections that aren't being closed properly. I've asked a few questions on here before about it, and it seems like everyone is telling me to use pool instead of the way I have been doing it. The only problem is that when I search online about using pool from promise-mysql, everyone seems to use a very simple and generic approach, but I'm using it within a complex application using sockets. So I'm wondering how I can switch my old approach using createConnection() to using pool instead, in hopes of clearing up these connection issues.
Each time I call a socket it makes a connection to the database and then releases it after it is complete, or so it seems. It sounds like this is not a very scalable approach, and that using pool will help run multiple queries in parallel.
db.js:
import mysql from 'promise-mysql';
import env from '../../../env.config.json';
const db = async (sql, descriptor, serializedParameters = []) => {
return new Promise( async (resolve, reject) => {
try {
const connection = await mysql.createConnection({
host: env.DB.HOST,
user: env.DB.USER,
password: env.DB.PASSWORD,
database: env.DB.NAME,
port: env.DB.PORT
})
if (connection && env.ENV === "development") {
//console.log(/*"There is a connection to the db for: ", descriptor*/);
}
let result;
if(serializedParameters.length > 0) {
result = await connection.query(sql, serializedParameters)
} else result = await connection.query(sql);
connection.end();
resolve(result);
} catch (e) {
console.log("ERROR pool.db: " + e);
reject(e);
};
});
}
export default db;
This is an example of how I would create a connection to query the db
inventory.js:
import db from '../API/db';
export const selectAllFromBuildItems = () => {
return new Promise( async (resolve, reject) => {
try {
const getAllBuildItems = "SELECT * FROM mydb.build_items;"
const response = await db(getAllBuildItems, "AllBuildItems");
resolve(response);
} catch (e) {
console.log("ERROR inventory.selectAllFromBuildItems: " + e);
reject(e);
}
});
};
How can I change my code so that I use a pool instead. I have a lot of different queries that can be called from our application so I'm not quite sure what the right approach for this would be. I saw some people say that I should create the pool once and then use it throughout the application, but I don't know where that would go. If anyone has any suggestions on how I can make this switch, that would help me out a lot. Thanks!
Create the pool. Better if you create once when you run your application.
If it is in different file then you have export here and import in required file.
var pool = mysql.createPool({
host: env.DB.HOST,
user: env.DB.USER,
password: env.DB.PASSWORD,
database: env.DB.NAME,
connectionLimit: 10
});
I had to create this prototype function as the library had a bug of close connection was not returning the connection to the pool.
pool.prototype.releaseConnection = function releaseConnection(connection) {
return this.pool.releaseConnection(connection.connection);
};
Funtion for getting connection from the pool that is created earlier.
If you want you can call pool.getConnection() in all your query functions.
function connect() {
return pool.getConnection().then(function(connection) {
return connection
}).catch(function(e) {
console.log("Error Creating Connection");
throw e;
});
}
Now this is your query function to get data from dd.
function selectAllFromBuildItems() {
var sql_query = `SELECT * FROM mydb.build_items`;
return connect().then(function(conn) {
return conn.query(sql_query).then(function(rows) {
pool.releaseConnection(conn);
return rows;
});
}).catch(function(e) {
console.log("ERROR inventory.selectAllFromBuildItems: " + e);
throw e;
});
}
Update: Descriptions are added. Hope this helps you.
I'm using node with mysql and I have a route that does:
const mysql = require("./mysql");
router.post("/register_user", (req, res) => {
mysql.register(req.body).then((result) => {
// stuff
});
});
mysql.js:
const mysql = require("mysql");
const connection = mysql.createConnection("mysql://...");
exports.register = (req) => {
const user = { name: req.name };
return new Promise((resolve, reject) => {
// make sure user doesn't exist already
connection.query('...', [user], (err, data) => {
...
if (isNewUser) {
connection.query('INSERT INTO USER...', user, (insertErr, rows) => {
...
resolve(rows);
connection.end();
}
}
});
});
}
This works perfectly when I register the first user in my app. But immediately after, if I log out (on the web app), then register a new user, I get an error saying:
Error: Cannot enqueue Query after invoking quit.
Why doesn't this create a new connection?
I assume you are using the following NPM module mysql
If it is the case then could you simply use MySQL pooling connections ?
Rather than creating and managing connections one-by-one, this module also provides built-in connection pooling using mysql.createPool(config).
So instead of calling connection.end(); you would be calling connection.release(); instead to return connection to the pool of open connections.