I am using Meteor.js, Apollo/GraphQL and a MySQL database with the mysql npm package as the driver.
I am creating a connection pool as follows:
import { createPool } from 'mysql';
const pool = createPool({
host: 'localhost',
user: 'root',
password: 'asdfasdf',
database: 'soundcraft',
});
export function query(sql) {
// const connection = this;
return new Promise((resolve, reject) => {
pool.query(sql, (err, results) => {
if (err) {
reject(err);
} else {
resolve(results);
}
});
});
}
I'm not sure what's happening but after a few database requests (equal to the connection limit) it will just freeze and give no error report, and it will stay frozen until the server is restarted.
EDIT: After some reading and finding this issue it doesn't seem have anything to do with the connection being released, rather it is the fact that the connection remains open by MySQL.
It seems impractical to just increase the connection limit as eventually surely the user will still use up their quota of connections.
It doesn't look like I'm approaching this problem in the right way
Related
I am trying to fix a bug in my node.js application, which is preventing a program I have written from ending successfully.
The script is returning the required results, but something is still running in the background which appears to be preventing it from terminating.
Through process of elimination, I have been able to narrow down the culprit to this getPlayers() function; but I am not sure why the problem is being caused.
Firstly, I am exporting the database pool from a module, like so:
require('dotenv').config()
const mysql = require("mysql"),
pool = mysql.createPool({
host: process.env.dbHost,
user: process.env.dbUser,
password: process.env.dbPassword,
database: process.env.dbDatabase
})
pool.connectionLimit = 15
module.exports = pool
And this is the query that appears to be causing all of the problems:
const getPlayers = () => {
return new Promise((resolve, reject) => {
db.query('SELECT * FROM cc_players', (err, res) => {
if(err)
console.log(err)
resolve(res)
})
})
}
If I comment out the db.query() function, and simply resolve an empty array, the script terminates as expected. Is there something about the database query that could be causing the script to continue running in the background?
Because you're creating a pool, previously opened SQL connections will not be closed, and instead kept for later uses. This is why Node.JS never exits.
To fix this, the mysql package provides a pool.end function to close all connections in the pool. You should call it when your script is ready to exit, perhaps like so:
function onExit() {
// ...
// Assuming ``db`` is the pool you created
db.end()
// ...
}
Beware that no further SQL operations can be performed on the pool after pool.end is called.
You created a pool, which will keep connections open for re-use. As long as there's open sockets, Node.js will not exit.
You have to use alternative way because create a pool will keep the connection opened in background, so you can use createConnection function instead of createPool then you can close the connection manually
let mysql = require('mysql');
let connection = mysql.createConnection({
host: process.env.dbHost,
user: process.env.dbUser,
password: process.env.dbPassword,
database: process.env.dbDatabase
});
const getPlayers = () => {
return new Promise((resolve, reject) => {
db.query('SELECT * FROM cc_players', (err, res) => {
if(err)
console.log(err)
resolve(res)
})
})
}
And once get the callback you can close the connection
getPlayer().then(res => {
.
.
.
connection.end();
})
Our stack is nodejs with MySQL we're using MySQL connections pooling our MySQL database is managed on AWS aurora .
in case of auto failover the master DB is changed the hostname stays the same but the connections inside the pool stays connected to the wrong DB.
The only why we found in order to reset the connection is to roll our servers.
this is a demonstration of a solution I think could solve this issue
but I prefer a solution without the set interval
const mysql = require('mysql');
class MysqlAdapter {
constructor() {
this.connectionType = 'MASTER';
this.waitingForAutoFaileOverSwitch = false;
this.poolCluster = mysql.createPoolCluster();
this.poolCluster.add(this.connectionType, {
host: 'localhost',
user: 'root',
password: 'root',
database: 'app'
});
this.intervalID = setInterval(() => {
if(this.waitingForAutoFaileOverSwitch) return;
this.excute('SHOW VARIABLES LIKE \'read_only\';').then(res => {
// if MASTER is set to read only is on then its mean a fail over is accoure and swe need to switch all connection in poll to secondry database
if (res[0].Value === 'ON') {
this.waitingForAutoFaileOverSwitch = true
this.poolCluster.end(() => {
this. waitingForAutoFaileOverSwitch = false
});
};
});
}, 5000);
}
async excute(query) {
// delay all incoming request until pool kill all connection to read only database
if (this.waitingForAutoFaileOverSwitch) {
return new Promise((resolve, reject) => {
setTimeout(() => {
this.excute(query).then(res => {
resolve(res);
});
}, 1000);
});
}
return new Promise((resolve, reject) => {
this.poolCluster.getConnection(this.connectionType, (err, connection) => {
if (err) {
reject(err);
}
connection.query(query, (err, rows) => {
connection.release();
if (err) {
reject(err);
}
resolve(rows);
});
});
});
}
}
const adapter = new MysqlAdapter();
Is there any other programmable way to reset the connection inside the pool?
Is there any notification we can listing to In case of auto-failover?
Instead of manually monitoring the DB health, as you have also hinted, ideally we subscribe to failover events published by AWS RDS Aurora.
There are multiple failover events listed here for the DB cluster: Amazon RDS event categories and event messages
You can use and test to see which one of them is the most reliable in your use case for triggering poolCluster.end() though.
My question is similar to this post but the solution didnt work for me probably because im using a different type of mysql connection (pool). This is my code:
let config= {
host: '***',
user: 'admin',
password: '***',
port: '3306',
database: '***',
multipleStatements: true
};
const con = mysql.createPool(config);
select();
function select(){
return new Promise((resolve, reject) => {
con.getConnection(function (err, connection) {
if (err) throw err;
else
console.log("Connected!");
let sql = "SELECT * FROM bidPrice WHERE idExchangePlatform = 2;";
connection.query(sql, function (err, results, fields) {
connection.release();
connection.destroy();
if (err) throw err;
console.log(results)
resolve(results);
});
});
});
}
I also important to mention that im running this function using the following command
node --max-old-space-size=31744 index.js # Increase to 31 GB
This is because im working with millions of records from the database query If i run this with regular node command i would be getting Javascript heap out of memory
When i tried integrating the solution i mentioned earlier to my code i just get a "killed" log after a while and then the process stops, should i handle server disconnect in a different way when using mysql.pool?
i'm learning node and have hit an issue with running a mysql query. I am certain i am getting connectivity to the mysql db (as when i run the query from the same module as i connect to the DB i am able to print the rows to the console log). However i get a 500 error when i require the DB connection into my another file and use in the query. I've spent last few hours on stack overflow and still stuck. Its definitely not the function as if i just return text it works fine so its definitely something around the way i am utilising the db connection. The DB connection file is below:
// initialize database connection
const mysql = require('mysql')
var db;
function connectDatabase() {
if (!db) {
db = mysql.createConnection({
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASS,
database: process.env.DB_DATABASE
})
db.connect(function(err){
if(!err) {
console.log('Database is connected!')
} else {
console.log('Error connecting database!')
}
})
}
return db
}
module.exports = connectDatabase()
The module where the DB connection us used in query below:
const db = require('../db').connectDatabase
exports.getNightlyRatesData = () => {
//const { rows } = db.query('SELECT * FROM rate', function (err, rows) {
db.query('SELECT * FROM rate', function (err, rows) {
if (err) {
console.log("ERROR");
console.log(err);
return;
}
console.log("good");
})
return rows
}
I use node js with mysql and want to avoid that the app crash on connection errors.At the moment i use this :
function mysql_handleDisconnect() {
mysql_connection = mysql.createConnection(mysql_config_obj); // Recreate the connection, since
// the old one cannot be reused.
mysql_connection.connect(function(err) { // The server is either down
if(err) { // or restarting (takes a while sometimes).
console.log('error when connecting to db:', err);
mysql_handleDisconnect(); // We introduce a delay before attempting to reconnect,
} // to avoid a hot loop, and to allow our node script to
}); // process asynchronous requests in the meantime.
// If you're also serving http, display a 503 error.
mysql_connection.on('error', function(err) {
console.log('db error', err);
if(err.code === 'PROTOCOL_CONNECTION_LOST') { // Connection to the MySQL server is usually
mysql_handleDisconnect(); // lost due to either server restart, or a
} else { // connnection idle timeout (the wait_timeout
throw err; // server variable configures this)
}
});
}
mysql_handleDisconnect(mysql_connection);
so this is blocking because it leads to a hot loop if the connection is closed.my problem is, if i add a setTimeout to reestablish connection just every 2 seconds i could get an fatal error when i do a query with "mysql_connection.query('SELECT ...')".in this case the app crashes.
So my question is,if there's a possibility to check the connection before i do a query?
Try using below code in every microservice before doing anything:
if(connection.state === 'disconnected'){
return respond(null, { status: 'fail', message: 'server down'});
}
State of connection to DB could fall in 2 states:
disconnected (when due to DB server down or wrong config use for DB connection is wrong)
authenticated (when DB connection is successfully created to DB server).
So either check state == 'disconnected' or state == 'authenticated'
I know this is an old question but I have found connection.ping( (err) => {...}) to be very useful for health-checks made from load balancers and whatnot.
Every time, while I'm pushing my code in production, the mysql connection is lost. It is a very common problem in production, or local.
My solution is that At every query established the db connection and remove connection after completing the db query.
My solution is to establish the db connection before every query, and then remove the connection after completing the db query.
Step1: Here is the code for dbConnection.js
//this code is for conenct to db
const mysql = require('mysql2');
require('dotenv').config();
module.exports.stablishedConnection = ()=>{
return new Promise((resolve,reject)=>{
const con = mysql.createConnection( {
host: process.env.DB_HOST||localhost,
user: process.env.DB_USER_NAME||myUserName ,
password: process.env.DB_PASSWORD||mypassword,
database: process.env.DB_NAME||mydb
});
con.connect((err) => {
if(err){
reject(err);
}
resolve(con);
});
})
}
module.exports.closeDbConnection =(con)=> {
con.destroy();
}
Step2: For Router.js I am import the db connection and handle the promise
const router = require('express').Router();
const {stablishedConnection,closeDbConnection} =require('../db/dbConnection');
router.get('/user/:sId/:userId',function(req,res){
stablishedConnection()
.then((db)=>{
console.log("Db connection stablished");
db.query(`select * from user WHERE sent_id=${req.params.sId} AND user_id=${req.params.userId}`, null, function (err,data) {
if (!data) {
res.status(200).json({sucess:false,err});
}else{
res.status(200).json({sucess:true,data});
closeDbConnection(db);
console.log("Db Connection close Successfully");
}
})
}).catch((error)=>{
console.log("Db not connected successfully",error);
});
});
router.get('/sen/:userId',function(req,res){
stablishedConnection()
.then((db)=>{
console.log("Db connection stablished");
db.query(`select * from sen WHERE user_id=${req.params.userId}`, null, function (err,data) {
if (!data) {
res.status(200).json({sucess:false,err});
}else{
res.status(200).json({sucess:true,data});
closeDbConnection(db);
console.log("Db Connection close Successfully");
}
})
}).catch((error)=>{
console.log("Db not connected successfully",error);
});
});
router.get('/language',(req,res)=>{
stablishedConnection()
.then((db)=>{
console.log("Db connection stablished");
db.query("select * from language", null, function (err,data) {
if (!data) {
res.status(200).json({sucess:false,err});
}else{
res.status(200).json({sucess:true,data});
closeDbConnection(db);
console.log("Db Connection close Successfully")
}
})
}).catch((error)=>{
console.log("Db not connected successfully",error);
});
})
module.exports = router;
This is perfectly run If you want to create and close connection at every query ..
I solved this problem like this:
let connection = mysql.createConnection(DB_CONFIG);
function runDBQuery() {
const disconnected = await new Promise(resolve => {
connection.ping(err => {
resolve(err);
});
});
if (disconnected) {
connection = mysql.createConnection(DB_CONFIG);
}
... use actual connection
}